Tinghong-Ye22 commited on
Commit
bcdb5ec
·
verified ·
1 Parent(s): 8562e9b

Add files using upload-large-folder tool

Browse files
This view is limited to 50 files because it contains too many changes.   See raw diff
Files changed (50) hide show
  1. nix/lib/runtime.nix +74 -0
  2. nix/runtime-base.nix +49 -0
  3. nix/runtime-host-gpu.nix +101 -0
  4. nix/runtime.nix +14 -0
  5. scripts/deepspeed/zero2.json +23 -0
  6. scripts/deepspeed/zero3.json +29 -0
  7. scripts/deepspeed/zero3_offload.json +56 -0
  8. scripts/infer/multiturn_auto_scenario_backchannel_train_v1.json +48 -0
  9. scripts/infer/multiturn_auto_scenario_backchannel_val_v1.json +48 -0
  10. scripts/infer/multiturn_auto_scenario_emergency_stop_train_v1.json +48 -0
  11. scripts/infer/multiturn_auto_scenario_emergency_stop_val_v1.json +40 -0
  12. scripts/infer/multiturn_auto_scenario_hesitation_filled_train_v2.json +60 -0
  13. scripts/infer/multiturn_auto_scenario_hesitation_filled_val_v2.json +60 -0
  14. scripts/infer/multiturn_auto_scenario_hesitation_repair_train_v1.json +42 -0
  15. scripts/infer/multiturn_auto_scenario_hesitation_repair_train_v2.json +63 -0
  16. scripts/infer/multiturn_auto_scenario_hesitation_repair_val_v1.json +42 -0
  17. scripts/infer/multiturn_auto_scenario_hesitation_repair_val_v2.json +62 -0
  18. scripts/infer/multiturn_auto_scenario_interruption_train_v1.json +60 -0
  19. scripts/infer/multiturn_auto_scenario_pause_train_v1.json +41 -0
  20. scripts/infer/multiturn_auto_scenario_pause_train_v2.json +60 -0
  21. scripts/infer/multiturn_auto_scenario_safe_hold_val_v1.json +40 -0
  22. scripts/infer/multiturn_auto_scenario_v1.json +61 -0
  23. scripts/infer/multiturn_auto_scenario_val_v1.json +100 -0
  24. scripts/infer/multiturn_auto_suite_v2.json +92 -0
  25. scripts/infer/multiturn_auto_suite_v3.json +92 -0
  26. scripts/infer/mux_full_duplex_audio.py +180 -0
  27. scripts/infer/render_third_party_review.py +287 -0
  28. scripts/infer/render_third_party_scale.py +151 -0
  29. scripts/infer/run_multiturn_infer_suite.py +700 -0
  30. scripts/infer/seed_audio_to_motion.py +530 -0
  31. scripts/infer/seed_motion_teacher_reconstruct.py +504 -0
  32. scripts/infer/seed_text_to_motion.py +873 -0
  33. scripts/infer/seed_video_to_motion.py +548 -0
  34. scripts/infer/third_party_checkpoint_probe.py +434 -0
  35. scripts/infer/third_party_scale_metrics.py +172 -0
  36. src/InternVLA-M1/InternVLA/__init__.py +0 -0
  37. src/InternVLA-M1/InternVLA/config/deepseeds/deepspeed_zero2.yaml +10 -0
  38. src/InternVLA-M1/InternVLA/config/deepseeds/ds_config.yaml +28 -0
  39. src/InternVLA-M1/InternVLA/config/deepseeds/zero2.yaml +21 -0
  40. src/InternVLA-M1/InternVLA/config/deepseeds/zero3.json +28 -0
  41. src/InternVLA-M1/InternVLA/config/training/internvla_cotrain_custom.yaml +126 -0
  42. src/InternVLA-M1/InternVLA/config/training/internvla_cotrain_humanoid.yaml +128 -0
  43. src/InternVLA-M1/InternVLA/config/training/internvla_cotrain_libero.yaml +126 -0
  44. src/InternVLA-M1/InternVLA/config/training/internvla_cotrain_oxe.yaml +100 -0
  45. src/InternVLA-M1/InternVLA/config/training/internvla_cotrain_sim_demo.yaml +124 -0
  46. src/InternVLA-M1/InternVLA/dataloader/__init__.py +63 -0
  47. src/InternVLA-M1/InternVLA/dataloader/gr00t_lerobot/README.md +0 -0
  48. src/InternVLA-M1/InternVLA/dataloader/gr00t_lerobot/__init__.py +0 -0
  49. src/InternVLA-M1/InternVLA/dataloader/gr00t_lerobot/data_config.py +770 -0
  50. src/InternVLA-M1/InternVLA/dataloader/gr00t_lerobot/datasets.py +2118 -0
nix/lib/runtime.nix ADDED
@@ -0,0 +1,74 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ { lib, pkgs }:
2
+
3
+ let
4
+ exportEnv = env:
5
+ lib.concatStringsSep "\n" (
6
+ lib.mapAttrsToList (name: value: "export ${name}=${lib.escapeShellArg value}") env
7
+ );
8
+
9
+ prependEnv = env:
10
+ lib.concatStringsSep "\n" (
11
+ lib.mapAttrsToList (
12
+ name: value:
13
+ ''
14
+ _runtime_current_value="$(printenv ${name} || true)"
15
+ if [ -n "$_runtime_current_value" ]; then
16
+ export ${name}=${lib.escapeShellArg value}:"$_runtime_current_value"
17
+ else
18
+ export ${name}=${lib.escapeShellArg value}
19
+ fi
20
+ unset _runtime_current_value
21
+ ''
22
+ ) env
23
+ );
24
+
25
+ mergeStringAttrs = runtimes: key:
26
+ lib.foldl' (
27
+ acc: runtime:
28
+ let current = runtime.${key} or { };
29
+ in acc // lib.mapAttrs (
30
+ name: value:
31
+ if builtins.hasAttr name acc
32
+ then "${value}:${builtins.getAttr name acc}"
33
+ else value
34
+ ) current
35
+ ) { } runtimes;
36
+
37
+ mergeRuntimes = runtimes: {
38
+ packages = lib.unique (lib.concatMap (runtime: runtime.packages or [ ]) runtimes);
39
+ env = lib.foldl' (acc: runtime: acc // (runtime.env or { })) { } runtimes;
40
+ prependEnv = mergeStringAttrs runtimes "prependEnv";
41
+ shellHook = lib.concatStringsSep "\n\n" (
42
+ lib.filter (chunk: chunk != "") (map (runtime: runtime.shellHook or "") runtimes)
43
+ );
44
+ };
45
+
46
+ mkRuntimeShell = {
47
+ name,
48
+ runtimes,
49
+ extraPackages ? [ ],
50
+ extraShellHook ? "",
51
+ stdenv ? null,
52
+ }:
53
+ let
54
+ runtime = mergeRuntimes runtimes;
55
+ shellHook = lib.concatStringsSep "\n\n" (
56
+ [
57
+ (exportEnv (runtime.env or { }))
58
+ (prependEnv (runtime.prependEnv or { }))
59
+ (runtime.shellHook or "")
60
+ extraShellHook
61
+ ''
62
+ echo "${name} shell ready"
63
+ ''
64
+ ]
65
+ );
66
+ in
67
+ pkgs.mkShell ({
68
+ packages = lib.unique (extraPackages ++ runtime.packages);
69
+ inherit shellHook;
70
+ } // lib.optionalAttrs (stdenv != null) { inherit stdenv; });
71
+ in
72
+ {
73
+ inherit exportEnv prependEnv mergeRuntimes mkRuntimeShell;
74
+ }
nix/runtime-base.nix ADDED
@@ -0,0 +1,49 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ { pkgs }:
2
+
3
+ let
4
+ cuda = pkgs.cudaPackages_12_8.cudatoolkit;
5
+ gcc = pkgs.gcc13;
6
+ ffmpeg = pkgs.ffmpeg_7;
7
+ ffmpegLib = ffmpeg.lib;
8
+ caBundle = "${pkgs.cacert}/etc/ssl/certs/ca-bundle.crt";
9
+ in
10
+ {
11
+ packages = [
12
+ pkgs.cmake
13
+ pkgs.ninja
14
+ pkgs.pkg-config
15
+ gcc
16
+ pkgs.cacert
17
+ pkgs.linuxHeaders
18
+ ffmpeg
19
+ cuda
20
+ ];
21
+
22
+ env = {
23
+ CUDA_HOME = "${cuda}";
24
+ CUDA_PATH = "${cuda}";
25
+ CUDACXX = "${cuda}/bin/nvcc";
26
+ CUDAHOSTCXX = "${gcc}/bin/g++";
27
+ CC = "${gcc}/bin/gcc";
28
+ CXX = "${gcc}/bin/g++";
29
+ CMAKE_CUDA_HOST_COMPILER = "${gcc}/bin/gcc";
30
+ NVCC_PREPEND_FLAGS = "-ccbin ${gcc}/bin/gcc";
31
+ SSL_CERT_FILE = caBundle;
32
+ NIX_SSL_CERT_FILE = caBundle;
33
+ REQUESTS_CA_BUNDLE = caBundle;
34
+ CURL_CA_BUNDLE = caBundle;
35
+ GIT_SSL_CAINFO = caBundle;
36
+ };
37
+
38
+ prependEnv = {
39
+ PATH = "${ffmpeg}/bin";
40
+ CPATH = "${pkgs.linuxHeaders}/include:${cuda}/include";
41
+ LIBRARY_PATH = "${cuda}/lib64:${cuda}/lib";
42
+ LD_LIBRARY_PATH = "${ffmpegLib}/lib:${cuda}/lib64:${cuda}/lib:${pkgs.stdenv.cc.cc.lib}/lib";
43
+ };
44
+
45
+ shellHook = ''
46
+ export CFLAGS="-I${pkgs.linuxHeaders}/include ''${CFLAGS:-}"
47
+ export CXXFLAGS="-I${pkgs.linuxHeaders}/include ''${CXXFLAGS:-}"
48
+ '';
49
+ }
nix/runtime-host-gpu.nix ADDED
@@ -0,0 +1,101 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ { pkgs }:
2
+
3
+ {
4
+ shellHook = ''
5
+ _runtime_detect_and_prepend() {
6
+ _runtime_var_name="$1"
7
+ shift
8
+ _runtime_joined_paths=""
9
+ for _runtime_path in "$@"; do
10
+ if [ -e "$_runtime_path" ]; then
11
+ if [ -n "$_runtime_joined_paths" ]; then
12
+ _runtime_joined_paths="$_runtime_joined_paths:$_runtime_path"
13
+ else
14
+ _runtime_joined_paths="$_runtime_path"
15
+ fi
16
+ fi
17
+ done
18
+
19
+ if [ -n "$_runtime_joined_paths" ]; then
20
+ eval "_runtime_current_value=\''${$_runtime_var_name}"
21
+ if [ -n "$_runtime_current_value" ]; then
22
+ export "$_runtime_var_name=$_runtime_joined_paths:$_runtime_current_value"
23
+ else
24
+ export "$_runtime_var_name=$_runtime_joined_paths"
25
+ fi
26
+ unset _runtime_current_value
27
+ fi
28
+
29
+ unset _runtime_joined_paths
30
+ unset _runtime_var_name
31
+ unset _runtime_path
32
+ }
33
+
34
+ if [ -z "''${VK_ICD_FILENAMES:-}" ]; then
35
+ for _runtime_vk_icd in \
36
+ /run/opengl-driver/share/vulkan/icd.d/nvidia_icd.x86_64.json \
37
+ /usr/share/vulkan/icd.d/nvidia_icd.json \
38
+ /usr/share/vulkan/icd.d/nvidia_icd.x86_64.json \
39
+ /etc/vulkan/icd.d/nvidia_icd.json
40
+ do
41
+ if [ -f "$_runtime_vk_icd" ]; then
42
+ export VK_ICD_FILENAMES="$_runtime_vk_icd"
43
+ break
44
+ fi
45
+ done
46
+ unset _runtime_vk_icd
47
+ fi
48
+
49
+ if [ -z "''${__EGL_VENDOR_LIBRARY_FILENAMES:-}" ]; then
50
+ for _runtime_egl_vendor in \
51
+ /run/opengl-driver/share/glvnd/egl_vendor.d/10_nvidia.json \
52
+ /usr/share/glvnd/egl_vendor.d/10_nvidia.json \
53
+ /usr/share/glvnd/egl_vendor.d/50_nvidia.json
54
+ do
55
+ if [ -f "$_runtime_egl_vendor" ]; then
56
+ export __EGL_VENDOR_LIBRARY_FILENAMES="$_runtime_egl_vendor"
57
+ break
58
+ fi
59
+ done
60
+ unset _runtime_egl_vendor
61
+ fi
62
+
63
+ _runtime_detect_and_prepend LD_LIBRARY_PATH \
64
+ /run/opengl-driver/lib \
65
+ /run/opengl-driver-32/lib \
66
+ /usr/lib/x86_64-linux-gnu/nvidia/current \
67
+ /usr/lib/x86_64-linux-gnu/nvidia \
68
+ /usr/lib/nvidia \
69
+ /usr/lib/nvidia-570 \
70
+ /usr/lib/nvidia-575 \
71
+ /usr/lib/nvidia-580 \
72
+ /usr/lib/nvidia-590 \
73
+ /usr/lib/wsl/lib
74
+
75
+ _runtime_detect_and_prepend XDG_DATA_DIRS \
76
+ /run/opengl-driver/share \
77
+ /usr/share
78
+
79
+ if [ -z "''${TRITON_LIBCUDA_PATH:-}" ]; then
80
+ for _runtime_triton_cuda in \
81
+ /run/opengl-driver/lib \
82
+ /usr/lib/x86_64-linux-gnu \
83
+ /usr/lib/x86_64-linux-gnu/nvidia/current \
84
+ /usr/lib/x86_64-linux-gnu/nvidia \
85
+ /usr/lib/nvidia-590 \
86
+ /usr/lib/nvidia-580 \
87
+ /usr/lib/nvidia-575 \
88
+ /usr/lib/nvidia-570 \
89
+ /usr/lib/wsl/lib
90
+ do
91
+ if [ -d "$_runtime_triton_cuda" ]; then
92
+ export TRITON_LIBCUDA_PATH="$_runtime_triton_cuda"
93
+ break
94
+ fi
95
+ done
96
+ unset _runtime_triton_cuda
97
+ fi
98
+
99
+ unset -f _runtime_detect_and_prepend
100
+ '';
101
+ }
nix/runtime.nix ADDED
@@ -0,0 +1,14 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ { pkgs }:
2
+
3
+ let
4
+ runtimeLib = import ./lib/runtime.nix {
5
+ lib = pkgs.lib;
6
+ inherit pkgs;
7
+ };
8
+ baseRuntime = import ./runtime-base.nix { inherit pkgs; };
9
+ hostGpuRuntime = import ./runtime-host-gpu.nix { inherit pkgs; };
10
+ in
11
+ runtimeLib.mergeRuntimes [
12
+ baseRuntime
13
+ hostGpuRuntime
14
+ ]
scripts/deepspeed/zero2.json ADDED
@@ -0,0 +1,23 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "fp16": {
3
+ "enabled": "auto",
4
+ "loss_scale": 0,
5
+ "loss_scale_window": 1000,
6
+ "initial_scale_power": 16,
7
+ "hysteresis": 2,
8
+ "min_loss_scale": 1
9
+ },
10
+ "bf16": {
11
+ "enabled": "auto"
12
+ },
13
+ "train_micro_batch_size_per_gpu": "auto",
14
+ "train_batch_size": "auto",
15
+ "gradient_accumulation_steps": "auto",
16
+ "zero_optimization": {
17
+ "stage": 2,
18
+ "overlap_comm": true,
19
+ "contiguous_gradients": true,
20
+ "sub_group_size": 1e9,
21
+ "reduce_bucket_size": "auto"
22
+ }
23
+ }
scripts/deepspeed/zero3.json ADDED
@@ -0,0 +1,29 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "fp16": {
3
+ "enabled": "auto",
4
+ "loss_scale": 0,
5
+ "loss_scale_window": 1000,
6
+ "initial_scale_power": 16,
7
+ "hysteresis": 2,
8
+ "min_loss_scale": 1
9
+ },
10
+ "bf16": {
11
+ "enabled": "auto"
12
+ },
13
+ "train_micro_batch_size_per_gpu": "auto",
14
+ "train_batch_size": "auto",
15
+ "gradient_accumulation_steps": "auto",
16
+ "gradient_clipping": 1.0,
17
+ "zero_optimization": {
18
+ "stage": 3,
19
+ "overlap_comm": true,
20
+ "contiguous_gradients": true,
21
+ "sub_group_size": 1e9,
22
+ "reduce_bucket_size": "auto",
23
+ "stage3_prefetch_bucket_size": "auto",
24
+ "stage3_param_persistence_threshold": "auto",
25
+ "stage3_max_live_parameters": 1e9,
26
+ "stage3_max_reuse_distance": 1e9,
27
+ "stage3_gather_16bit_weights_on_model_save": true
28
+ }
29
+ }
scripts/deepspeed/zero3_offload.json ADDED
@@ -0,0 +1,56 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "fp16": {
3
+ "enabled": "auto",
4
+ "loss_scale": 0,
5
+ "loss_scale_window": 1000,
6
+ "initial_scale_power": 16,
7
+ "hysteresis": 2,
8
+ "min_loss_scale": 1
9
+ },
10
+ "bf16": {
11
+ "enabled": "auto"
12
+ },
13
+ "optimizer": {
14
+ "type": "AdamW",
15
+ "params": {
16
+ "lr": "auto",
17
+ "betas": "auto",
18
+ "eps": "auto",
19
+ "weight_decay": "auto"
20
+ }
21
+ },
22
+ "scheduler": {
23
+ "type": "WarmupLR",
24
+ "params": {
25
+ "warmup_min_lr": "auto",
26
+ "warmup_max_lr": "auto",
27
+ "warmup_num_steps": "auto"
28
+ }
29
+ },
30
+ "zero_optimization": {
31
+ "stage": 3,
32
+ "offload_optimizer": {
33
+ "device": "cpu",
34
+ "pin_memory": true
35
+ },
36
+ "offload_param": {
37
+ "device": "cpu",
38
+ "pin_memory": true
39
+ },
40
+ "overlap_comm": true,
41
+ "contiguous_gradients": true,
42
+ "sub_group_size": 1e9,
43
+ "reduce_bucket_size": "auto",
44
+ "stage3_prefetch_bucket_size": "auto",
45
+ "stage3_param_persistence_threshold": "auto",
46
+ "stage3_max_live_parameters": 1e9,
47
+ "stage3_max_reuse_distance": 1e9,
48
+ "gather_16bit_weights_on_model_save": true
49
+ },
50
+ "gradient_accumulation_steps": "auto",
51
+ "gradient_clipping": "auto",
52
+ "train_batch_size": "auto",
53
+ "train_micro_batch_size_per_gpu": "auto",
54
+ "steps_per_print": 1e5,
55
+ "wall_clock_breakdown": false
56
+ }
scripts/infer/multiturn_auto_scenario_backchannel_train_v1.json ADDED
@@ -0,0 +1,48 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "format": "hfd_multiturn_auto_infer_scenario_v1",
3
+ "name": "backchannel_grab_mmhmm_wave_zh",
4
+ "interaction_mode": "backchannel",
5
+ "data_split": "train",
6
+ "split_seed": 42,
7
+ "val_ratio": 0.1,
8
+ "split_source_namespace": "seed",
9
+ "description": "第一条动作执行中插入同音色的训练集短反馈“嗯嗯”,动作必须继续;随后再给一条真正的新指令。",
10
+ "required_semantics_version": 22,
11
+ "initial_idle_steps": 5,
12
+ "initial_idle_seconds": 2.0,
13
+ "motion_frames_per_turn": 450,
14
+ "motion_seconds_per_turn": 9.0,
15
+ "inter_turn_background_steps_by_turn": [10, 15],
16
+ "inter_turn_background_seconds_by_turn": [4.0, 6.0],
17
+ "terminal_background_steps": 5,
18
+ "terminal_background_seconds": 2.0,
19
+ "turns": [
20
+ {
21
+ "turn_index": 1,
22
+ "source_motion_filename": "grab_walk_ff_180_001__A548",
23
+ "audio_path": "/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/qwen3-tts/seed_audio/000006/female_01_zh.wav",
24
+ "spoken_command": "向前走并抓取某物",
25
+ "ideal_motion": "听完后向前走并抓取,动作持续执行。"
26
+ },
27
+ {
28
+ "turn_index": 2,
29
+ "backchannel_intent": "continuer",
30
+ "audio_path": "/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/videogen/hfd_backchannel_commands_v2_short/audio/train/bc_cont_01/female_01_zh.wav",
31
+ "spoken_command": "嗯嗯",
32
+ "ideal_motion": "这是短反馈而不是新指令;控制持续输出 </motion>,抓取动作的相位、RTC 和 future progress 均连续,不进入 listen 或 idle。"
33
+ },
34
+ {
35
+ "turn_index": 3,
36
+ "source_motion_filename": "wave_two_hands_R_001__A533",
37
+ "audio_path": "/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/qwen3-tts/seed_audio/000155/female_01_zh.wav",
38
+ "spoken_command": "双手在身前挥动",
39
+ "ideal_motion": "这是真正的新指令,模型才应进入 listen,随后从当前状态执行双手挥动。"
40
+ }
41
+ ],
42
+ "ideal_control_timeline": [
43
+ "第一条动作指令遵循 idle→listen→motion",
44
+ "动作中听到“嗯嗯”仍保持 motion,不重置动作相位",
45
+ "后续真正的新指令到来后才从 motion 切到 listen,再进入新 motion",
46
+ "末尾纯背景回到 idle"
47
+ ]
48
+ }
scripts/infer/multiturn_auto_scenario_backchannel_val_v1.json ADDED
@@ -0,0 +1,48 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "format": "hfd_multiturn_auto_infer_scenario_v1",
3
+ "name": "backchannel_jog_indeed_pick_en",
4
+ "interaction_mode": "backchannel",
5
+ "data_split": "val",
6
+ "split_seed": 42,
7
+ "val_ratio": 0.1,
8
+ "split_source_namespace": "seed",
9
+ "description": "未见过的验证动作与验证短反馈组合;检查 backchannel 的语义泛化,而不是记住固定词。",
10
+ "required_semantics_version": 22,
11
+ "initial_idle_steps": 4,
12
+ "initial_idle_seconds": 1.6,
13
+ "motion_frames_per_turn": 450,
14
+ "motion_seconds_per_turn": 9.0,
15
+ "inter_turn_background_steps_by_turn": [9, 16],
16
+ "inter_turn_background_seconds_by_turn": [3.6, 6.4],
17
+ "terminal_background_steps": 6,
18
+ "terminal_background_seconds": 2.4,
19
+ "turns": [
20
+ {
21
+ "turn_index": 1,
22
+ "source_motion_filename": "jog_ff_start_180_R_003__A068",
23
+ "audio_path": "/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/qwen3-tts/seed_audio/000175/female_02_en.wav",
24
+ "spoken_command": "start jog forward",
25
+ "ideal_motion": "听完后向前慢跑,并保持连续闭环状态。"
26
+ },
27
+ {
28
+ "turn_index": 2,
29
+ "backchannel_intent": "continuer",
30
+ "audio_path": "/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/videogen/hfd_backchannel_commands_v2_short/audio/val/bc_cont_11/female_02_en.wav",
31
+ "spoken_command": "Indeed.",
32
+ "ideal_motion": "验证集短反馈不应抢占控制权;慢跑动作、RTC、进度和 Qwen KV 均连续。"
33
+ },
34
+ {
35
+ "turn_index": 3,
36
+ "source_motion_filename": "medium_big_light_two_hands_right_side_low_to_right_side_high_R_001__A529",
37
+ "audio_path": "/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/qwen3-tts/seed_audio/000227/female_02_en.wav",
38
+ "spoken_command": "picks up medium big light object with two hands from low right side and puts it away to the right side high",
39
+ "ideal_motion": "真正的新指令到来后才切入 listen,随后平滑执行右侧双手取放动作。"
40
+ }
41
+ ],
42
+ "ideal_control_timeline": [
43
+ "初始背景输出 idle,第一条动作指令进入 listen 后执行 motion",
44
+ "未见过的短反馈期间持续输出 motion,不出现假 interruption",
45
+ "下一条真实指令出现后才切换 listen 和新 motion",
46
+ "最后在背景中回到 idle"
47
+ ]
48
+ }
scripts/infer/multiturn_auto_scenario_emergency_stop_train_v1.json ADDED
@@ -0,0 +1,48 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "format": "hfd_multiturn_auto_infer_scenario_v1",
3
+ "name": "stop_train_grab_stop_wave_zh",
4
+ "interaction_mode": "emergency-stop",
5
+ "data_split": "train",
6
+ "split_seed": 42,
7
+ "val_ratio": 0.1,
8
+ "split_source_namespace": "seed",
9
+ "description": "训练集抓取行走执行中收到中文急停,停住并保持两秒背景后再接挥手指令,检查控制后的恢复交互。",
10
+ "required_semantics_version": 21,
11
+ "initial_idle_steps": 5,
12
+ "initial_idle_seconds": 2.0,
13
+ "motion_frames_per_turn": 450,
14
+ "motion_seconds_per_turn": 9.0,
15
+ "inter_turn_background_steps_by_turn": [8, 5],
16
+ "inter_turn_background_seconds_by_turn": [3.2, 2.0],
17
+ "terminal_background_steps": 5,
18
+ "terminal_background_seconds": 2.0,
19
+ "turns": [
20
+ {
21
+ "turn_index": 1,
22
+ "source_motion_filename": "grab_walk_ff_180_001__A548",
23
+ "audio_path": "/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/qwen3-tts/seed_audio/000006/female_01_zh.wav",
24
+ "spoken_command": "向前走并抓取某物",
25
+ "ideal_motion": "听完后向前行走并开始抓取。"
26
+ },
27
+ {
28
+ "turn_index": 2,
29
+ "control_intent": "emergency-stop",
30
+ "audio_path": "/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/videogen/hfd_control_commands_v1/audio/emergency-stop/stop_02/female_01_zh.wav",
31
+ "spoken_command": "马上停下来",
32
+ "ideal_motion": "完整听到急停后直接从 motion 切到 idle,平滑停住,不出现 listen。"
33
+ },
34
+ {
35
+ "turn_index": 3,
36
+ "source_motion_filename": "wave_two_hands_R_001__A533",
37
+ "audio_path": "/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/qwen3-tts/seed_audio/000155/female_01_zh.wav",
38
+ "spoken_command": "双手在身前挥动",
39
+ "ideal_motion": "急停后保持约两秒,再正常进入 listen 并执行双手挥动。"
40
+ }
41
+ ],
42
+ "ideal_control_timeline": [
43
+ "初始背景为 </idle>,第一条动作指令期间为 </listen>",
44
+ "执行 </motion> 约 3.2 秒后收到急停语音",
45
+ "急停语音未听完前仍是 </motion>,听完后直接转 </idle>",
46
+ "保持约两秒连续背景后接收新指令,重新进入 </listen> → </motion>"
47
+ ]
48
+ }
scripts/infer/multiturn_auto_scenario_emergency_stop_val_v1.json ADDED
@@ -0,0 +1,40 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "format": "hfd_multiturn_auto_infer_scenario_v1",
3
+ "name": "stop_val_jog_en",
4
+ "interaction_mode": "emergency-stop",
5
+ "data_split": "val",
6
+ "split_seed": 42,
7
+ "val_ratio": 0.1,
8
+ "split_source_namespace": "seed",
9
+ "description": "未见过的慢跑 motion 执行中收到英文急停,检查特殊指令的泛化。",
10
+ "required_semantics_version": 21,
11
+ "initial_idle_steps": 4,
12
+ "initial_idle_seconds": 1.6,
13
+ "motion_frames_per_turn": 450,
14
+ "motion_seconds_per_turn": 9.0,
15
+ "inter_turn_background_steps_by_turn": [9],
16
+ "inter_turn_background_seconds_by_turn": [3.6],
17
+ "terminal_background_steps": 6,
18
+ "terminal_background_seconds": 2.4,
19
+ "turns": [
20
+ {
21
+ "turn_index": 1,
22
+ "source_motion_filename": "jog_ff_start_180_R_003__A068",
23
+ "audio_path": "/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/qwen3-tts/seed_audio/000175/female_02_en.wav",
24
+ "spoken_command": "start jog forward",
25
+ "ideal_motion": "开始向前慢跑。"
26
+ },
27
+ {
28
+ "turn_index": 2,
29
+ "control_intent": "emergency-stop",
30
+ "audio_path": "/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/videogen/hfd_control_commands_v1/audio/emergency-stop/stop_02/female_02_en.wav",
31
+ "spoken_command": "Stop now.",
32
+ "ideal_motion": "完整听到 stop 后直接平滑停住,不进入 listen。"
33
+ }
34
+ ],
35
+ "ideal_control_timeline": [
36
+ "idle -> listen -> motion",
37
+ "急停语音在 motion 中到来",
38
+ "听清后 motion -> idle,中间没有 listen"
39
+ ]
40
+ }
scripts/infer/multiturn_auto_scenario_hesitation_filled_train_v2.json ADDED
@@ -0,0 +1,60 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "format": "hfd_multiturn_auto_infer_scenario_v1",
3
+ "name": "filled_wave_then_left_zh",
4
+ "interaction_mode": "hesitation",
5
+ "interaction_subtype": "filled-pause",
6
+ "data_split": "train",
7
+ "split_seed": 42,
8
+ "val_ratio": 0.1,
9
+ "split_source_namespace": "seed",
10
+ "description": "训练集中文指令中插入同音色、自然拖长的“呃……”,实际不确定区间为 2.4 秒,最终意图不变。",
11
+ "required_semantics_version": 24,
12
+ "initial_idle_steps": 5,
13
+ "initial_idle_seconds": 2.0,
14
+ "motion_frames_per_turn": 450,
15
+ "motion_seconds_per_turn": 9.0,
16
+ "inter_turn_background_steps_by_turn": [
17
+ 25
18
+ ],
19
+ "inter_turn_background_seconds_by_turn": [
20
+ 10.0
21
+ ],
22
+ "terminal_background_steps": 5,
23
+ "terminal_background_seconds": 2.0,
24
+ "disfluency_event": {
25
+ "turn_index": 0,
26
+ "subtype": "filled-pause",
27
+ "gap_steps": 6,
28
+ "gap_seconds": 2.4,
29
+ "marker_id": "hes_fill_zh_02"
30
+ },
31
+ "turns": [
32
+ {
33
+ "turn_index": 1,
34
+ "disfluency_subtype": "filled-pause",
35
+ "source_motion_filename": "wave_two_hands_R_001__A533",
36
+ "audio_path": "/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/videogen/hfd_hesitation_pause_v2/audio/review_examples/train/filled_wave_zh_female_01_zh.wav",
37
+ "spoken_command": "双手在身前……呃……挥动",
38
+ "final_instruction": "双手在身前挥动",
39
+ "ideal_motion": "听到呃时继续 listen,不提前抬手;完整指令结束后再完成双手挥动。"
40
+ },
41
+ {
42
+ "turn_index": 2,
43
+ "source_motion_filename": "neutral_walk_ff_270_R_001__A534_M",
44
+ "audio_path": "/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/qwen3-tts/seed_audio/000120/female_01_zh.wav",
45
+ "spoken_command": "向左走路",
46
+ "ideal_motion": "随后相对当前朝向向左走。"
47
+ }
48
+ ],
49
+ "ideal_control_timeline": [
50
+ "背景 idle",
51
+ "前半句、呃和后半句均为 listen",
52
+ "只有最终意图完整后才进入 motion",
53
+ "第二轮正常执行"
54
+ ],
55
+ "continuity_requirements": [
56
+ "同一女性音色和语言",
57
+ "同一 Qwen KV 与连续背景",
58
+ "RTC 和动作相位不在犹豫处重置"
59
+ ]
60
+ }
scripts/infer/multiturn_auto_scenario_hesitation_filled_val_v2.json ADDED
@@ -0,0 +1,60 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "format": "hfd_multiturn_auto_infer_scenario_v1",
3
+ "name": "filled_pick_then_jog_en",
4
+ "interaction_mode": "hesitation",
5
+ "interaction_subtype": "filled-pause",
6
+ "data_split": "val",
7
+ "split_seed": 42,
8
+ "val_ratio": 0.1,
9
+ "split_source_namespace": "seed",
10
+ "description": "验证集英文长指令中插入未用于训练的同音色、自然拖长的 Hmm,检查长语义在 KV 中继续累积。",
11
+ "required_semantics_version": 24,
12
+ "initial_idle_steps": 4,
13
+ "initial_idle_seconds": 1.6,
14
+ "motion_frames_per_turn": 450,
15
+ "motion_seconds_per_turn": 9.0,
16
+ "inter_turn_background_steps_by_turn": [
17
+ 25
18
+ ],
19
+ "inter_turn_background_seconds_by_turn": [
20
+ 10.0
21
+ ],
22
+ "terminal_background_steps": 6,
23
+ "terminal_background_seconds": 2.4,
24
+ "disfluency_event": {
25
+ "turn_index": 0,
26
+ "subtype": "filled-pause",
27
+ "gap_steps": 5,
28
+ "gap_seconds": 2.0,
29
+ "marker_id": "hes_fill_en_11"
30
+ },
31
+ "turns": [
32
+ {
33
+ "turn_index": 1,
34
+ "disfluency_subtype": "filled-pause",
35
+ "source_motion_filename": "medium_big_light_two_hands_right_side_low_to_right_side_high_R_001__A529",
36
+ "audio_path": "/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/videogen/hfd_hesitation_pause_v2/audio/review_examples/val/filled_pick_en_female_02_en.wav",
37
+ "spoken_command": "picks up ... from low ... Hmm ... right side and puts it away to the right side high",
38
+ "final_instruction": "picks up medium big light object with two hands from low right side and puts it away to the right side high",
39
+ "ideal_motion": "Hmm 前后都保持 listen,完整结束后才执行右下到右上的双手取放。"
40
+ },
41
+ {
42
+ "turn_index": 2,
43
+ "source_motion_filename": "jog_ff_start_180_R_003__A068",
44
+ "audio_path": "/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/qwen3-tts/seed_audio/000175/female_02_en.wav",
45
+ "spoken_command": "start jog forward",
46
+ "ideal_motion": "随后从当前状态向前慢跑。"
47
+ }
48
+ ],
49
+ "ideal_control_timeline": [
50
+ "初始 idle",
51
+ "整条含 Hmm 的未完成指令保持 listen",
52
+ "完整语义后进入 motion",
53
+ "第二轮正常执行"
54
+ ],
55
+ "continuity_requirements": [
56
+ "验证 marker 与训练 marker 含义拆分",
57
+ "同一音色、KV、背景和 RTC",
58
+ "两条 motion source 均属于 val"
59
+ ]
60
+ }
scripts/infer/multiturn_auto_scenario_hesitation_repair_train_v1.json ADDED
@@ -0,0 +1,42 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "format": "hfd_multiturn_auto_infer_scenario_v1",
3
+ "name": "repair_grab_to_left_then_wave_zh",
4
+ "interaction_mode": "hesitation",
5
+ "interaction_subtype": "self-repair",
6
+ "data_split": "train",
7
+ "split_seed": 42,
8
+ "val_ratio": 0.1,
9
+ "split_source_namespace": "seed",
10
+ "description": "先说出抓取指令的不完整前缀,再用同音色明确改口为完整向左走;只能执行最终指令。",
11
+ "required_semantics_version": 23,
12
+ "initial_idle_steps": 5,
13
+ "initial_idle_seconds": 2.0,
14
+ "motion_frames_per_turn": 450,
15
+ "motion_seconds_per_turn": 9.0,
16
+ "inter_turn_background_steps_by_turn": [25],
17
+ "inter_turn_background_seconds_by_turn": [10.0],
18
+ "terminal_background_steps": 5,
19
+ "terminal_background_seconds": 2.0,
20
+ "disfluency_event": {"turn_index": 0, "subtype": "self-repair", "gap_steps": 7, "gap_seconds": 2.8, "marker_id": "repair_zh_01"},
21
+ "turns": [
22
+ {
23
+ "turn_index": 1,
24
+ "disfluency_subtype": "self-repair",
25
+ "source_motion_filename": "neutral_walk_ff_270_R_001__A534_M",
26
+ "audio_path": "/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/videogen/hfd_hesitation_pause_v1/audio/review_examples/train/repair_grab_to_left_zh_female_01_zh.wav",
27
+ "spoken_command": "向前……不对,我是说……向左走路",
28
+ "provisional_instruction": "向前走并抓取某物",
29
+ "final_instruction": "向左走路",
30
+ "ideal_motion": "整个改口过程只 listen;不得向前抓取,最终只相对当前朝向向左走。"
31
+ },
32
+ {
33
+ "turn_index": 2,
34
+ "source_motion_filename": "wave_two_hands_R_001__A533",
35
+ "audio_path": "/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/qwen3-tts/seed_audio/000155/female_01_zh.wav",
36
+ "spoken_command": "双手在身前挥动",
37
+ "ideal_motion": "第二轮正常挥手,验证最终意图替换没有破坏后续交互。"
38
+ }
39
+ ],
40
+ "ideal_control_timeline": ["初始 idle", "错误前缀、改口词和完整新指令全程 listen", "只在完整最终指令后进入 motion", "不为错误前缀启动 motion"],
41
+ "continuity_requirements": ["旧意图与最终意图共用同一 Qwen KV,不靠清缓存作弊", "同一音色和背景", "动作与 RTC 只从最终意图开始"]
42
+ }
scripts/infer/multiturn_auto_scenario_hesitation_repair_train_v2.json ADDED
@@ -0,0 +1,63 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "format": "hfd_multiturn_auto_infer_scenario_v1",
3
+ "name": "repair_grab_to_left_then_wave_zh",
4
+ "interaction_mode": "hesitation",
5
+ "interaction_subtype": "self-repair",
6
+ "data_split": "train",
7
+ "split_seed": 42,
8
+ "val_ratio": 0.1,
9
+ "split_source_namespace": "seed",
10
+ "description": "先说出抓取指令的不完整前缀,再用同音色明确改口为完整向左走;只能执行最终指令。",
11
+ "required_semantics_version": 24,
12
+ "initial_idle_steps": 5,
13
+ "initial_idle_seconds": 2.0,
14
+ "motion_frames_per_turn": 450,
15
+ "motion_seconds_per_turn": 9.0,
16
+ "inter_turn_background_steps_by_turn": [
17
+ 25
18
+ ],
19
+ "inter_turn_background_seconds_by_turn": [
20
+ 10.0
21
+ ],
22
+ "terminal_background_steps": 5,
23
+ "terminal_background_seconds": 2.0,
24
+ "disfluency_event": {
25
+ "turn_index": 0,
26
+ "subtype": "self-repair",
27
+ "gap_steps": 7,
28
+ "gap_seconds": 2.8,
29
+ "marker_id": "repair_zh_01",
30
+ "repair_style": "explicit-cue"
31
+ },
32
+ "turns": [
33
+ {
34
+ "turn_index": 1,
35
+ "disfluency_subtype": "self-repair",
36
+ "source_motion_filename": "neutral_walk_ff_270_R_001__A534_M",
37
+ "audio_path": "/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/videogen/hfd_hesitation_pause_v2/audio/review_examples/train/repair_grab_to_left_zh_female_01_zh.wav",
38
+ "spoken_command": "向前……不对,我是说……向左走路",
39
+ "provisional_instruction": "向前走并抓取某物",
40
+ "final_instruction": "向左走路",
41
+ "ideal_motion": "整个改口过程只 listen;不得向前抓取,最终只相对当前朝向向左走。",
42
+ "repair_style": "explicit-cue"
43
+ },
44
+ {
45
+ "turn_index": 2,
46
+ "source_motion_filename": "wave_two_hands_R_001__A533",
47
+ "audio_path": "/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/qwen3-tts/seed_audio/000155/female_01_zh.wav",
48
+ "spoken_command": "双手在身前挥动",
49
+ "ideal_motion": "第二轮正常挥手,验证最终意图替换没有破坏后续交互。"
50
+ }
51
+ ],
52
+ "ideal_control_timeline": [
53
+ "初始 idle",
54
+ "错误前缀、改口词和完整新指令全程 listen",
55
+ "只在完整最终指令后进入 motion",
56
+ "不为错误前缀启动 motion"
57
+ ],
58
+ "continuity_requirements": [
59
+ "旧意图与最终意图共用同一 Qwen KV,不靠清缓存作弊",
60
+ "同一音色和背景",
61
+ "动作与 RTC 只从最终意图开始"
62
+ ]
63
+ }
scripts/infer/multiturn_auto_scenario_hesitation_repair_val_v1.json ADDED
@@ -0,0 +1,42 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "format": "hfd_multiturn_auto_infer_scenario_v1",
3
+ "name": "repair_pick_to_jog_then_pick_en",
4
+ "interaction_mode": "hesitation",
5
+ "interaction_subtype": "self-repair",
6
+ "data_split": "val",
7
+ "split_seed": 42,
8
+ "val_ratio": 0.1,
9
+ "split_source_namespace": "seed",
10
+ "description": "未见 motion 上先说搬运长指令的错误前缀,再用验证集 cue 改口为完整向前慢跑。",
11
+ "required_semantics_version": 23,
12
+ "initial_idle_steps": 4,
13
+ "initial_idle_seconds": 1.6,
14
+ "motion_frames_per_turn": 450,
15
+ "motion_seconds_per_turn": 9.0,
16
+ "inter_turn_background_steps_by_turn": [25],
17
+ "inter_turn_background_seconds_by_turn": [10.0],
18
+ "terminal_background_steps": 6,
19
+ "terminal_background_seconds": 2.4,
20
+ "disfluency_event": {"turn_index": 0, "subtype": "self-repair", "gap_steps": 7, "gap_seconds": 2.8, "marker_id": "repair_en_11"},
21
+ "turns": [
22
+ {
23
+ "turn_index": 1,
24
+ "disfluency_subtype": "self-repair",
25
+ "source_motion_filename": "jog_ff_start_180_R_003__A068",
26
+ "audio_path": "/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/videogen/hfd_hesitation_pause_v1/audio/review_examples/val/repair_pick_to_jog_en_female_02_en.wav",
27
+ "spoken_command": "picks up ... Wait, I mean ... start jog forward",
28
+ "provisional_instruction": "picks up medium big light object with two hands from low right side and puts it away to the right side high",
29
+ "final_instruction": "start jog forward",
30
+ "ideal_motion": "不得执行搬运;完整听完 start jog forward 后才向前慢跑。"
31
+ },
32
+ {
33
+ "turn_index": 2,
34
+ "source_motion_filename": "medium_big_light_two_hands_right_side_low_to_right_side_high_R_001__A529",
35
+ "audio_path": "/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/qwen3-tts/seed_audio/000227/female_02_en.wav",
36
+ "spoken_command": "picks up medium big light object with two hands from low right side and puts it away to the right side high",
37
+ "ideal_motion": "只有第二轮真正完整重发搬运指令时,才执行右侧双手取放。"
38
+ }
39
+ ],
40
+ "ideal_control_timeline": ["初始 idle", "假开头、Wait I mean 和完整最终指令全程 listen", "第一轮只执行 jog", "第二轮完整指令后才执行 pick"],
41
+ "continuity_requirements": ["验证 cue 与训练 cue 词面分离", "Qwen KV 保留旧指令但最新完整意图应覆盖它", "两条动作与同一声音身份均保持 val 审计"]
42
+ }
scripts/infer/multiturn_auto_scenario_hesitation_repair_val_v2.json ADDED
@@ -0,0 +1,62 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "format": "hfd_multiturn_auto_infer_scenario_v1",
3
+ "name": "repair_pick_to_jog_then_pick_en",
4
+ "interaction_mode": "hesitation",
5
+ "interaction_subtype": "self-repair",
6
+ "data_split": "val",
7
+ "split_seed": 42,
8
+ "val_ratio": 0.1,
9
+ "split_source_namespace": "seed",
10
+ "description": "未见 motion 上先说搬运长指令的错误前缀,停顿后不使用 I mean 等提示词,直接重新说完整向前慢跑指令。",
11
+ "required_semantics_version": 24,
12
+ "initial_idle_steps": 4,
13
+ "initial_idle_seconds": 1.6,
14
+ "motion_frames_per_turn": 450,
15
+ "motion_seconds_per_turn": 9.0,
16
+ "inter_turn_background_steps_by_turn": [
17
+ 25
18
+ ],
19
+ "inter_turn_background_seconds_by_turn": [
20
+ 10.0
21
+ ],
22
+ "terminal_background_steps": 6,
23
+ "terminal_background_seconds": 2.4,
24
+ "disfluency_event": {
25
+ "turn_index": 0,
26
+ "subtype": "self-repair",
27
+ "gap_steps": 7,
28
+ "gap_seconds": 2.8,
29
+ "repair_style": "implicit-restart"
30
+ },
31
+ "turns": [
32
+ {
33
+ "turn_index": 1,
34
+ "disfluency_subtype": "self-repair",
35
+ "source_motion_filename": "jog_ff_start_180_R_003__A068",
36
+ "audio_path": "/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/videogen/hfd_hesitation_pause_v2/audio/review_examples/val/repair_pick_to_jog_en_female_02_en.wav",
37
+ "spoken_command": "picks up ... (2.8 s pause) ... start jog forward",
38
+ "provisional_instruction": "picks up medium big light object with two hands from low right side and puts it away to the right side high",
39
+ "final_instruction": "start jog forward",
40
+ "ideal_motion": "没有改口提示词也不得执行搬运;完整听完 start jog forward 后才向前慢跑。",
41
+ "repair_style": "implicit-restart"
42
+ },
43
+ {
44
+ "turn_index": 2,
45
+ "source_motion_filename": "medium_big_light_two_hands_right_side_low_to_right_side_high_R_001__A529",
46
+ "audio_path": "/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/qwen3-tts/seed_audio/000227/female_02_en.wav",
47
+ "spoken_command": "picks up medium big light object with two hands from low right side and puts it away to the right side high",
48
+ "ideal_motion": "只有第二轮真正完整重发搬运指令时,才执行右侧双手取放。"
49
+ }
50
+ ],
51
+ "ideal_control_timeline": [
52
+ "初始 idle",
53
+ "错误开头、停顿和重新开始的完整最终指令全程 listen",
54
+ "第一轮只执行 jog",
55
+ "第二轮完整指令后才执行 pick"
56
+ ],
57
+ "continuity_requirements": [
58
+ "不依赖 I mean 等关键词判断改口",
59
+ "Qwen KV 保留旧指令但最新完整意图应覆盖它",
60
+ "两条动作与同一声音身份均保持 val 审计"
61
+ ]
62
+ }
scripts/infer/multiturn_auto_scenario_interruption_train_v1.json ADDED
@@ -0,0 +1,60 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "format": "hfd_multiturn_auto_infer_scenario_v1",
3
+ "name": "interrupt_grab_wave_walk_left_zh",
4
+ "interaction_mode": "interruption",
5
+ "data_split": "train",
6
+ "split_seed": 42,
7
+ "val_ratio": 0.1,
8
+ "split_source_namespace": "seed",
9
+ "description": "三轮训练集指令;第二、三条语音分别在上一动作开始约 4.0/4.8 秒后到来,用于检查因果打断、连续 KV 和连续 RTC。",
10
+ "required_semantics_version": 20,
11
+ "initial_idle_steps": 5,
12
+ "initial_idle_seconds": 2.0,
13
+ "motion_frames_per_turn": 450,
14
+ "motion_seconds_per_turn": 9.0,
15
+ "inter_turn_background_steps_by_turn": [
16
+ 10,
17
+ 12
18
+ ],
19
+ "inter_turn_background_seconds_by_turn": [
20
+ 4.0,
21
+ 4.8
22
+ ],
23
+ "terminal_background_steps": 5,
24
+ "terminal_background_seconds": 2.0,
25
+ "turns": [
26
+ {
27
+ "turn_index": 1,
28
+ "source_motion_filename": "grab_walk_ff_180_001__A548",
29
+ "audio_path": "/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/qwen3-tts/seed_audio/000006/female_01_zh.wav",
30
+ "spoken_command": "向前走并抓取某物",
31
+ "ideal_motion": "听完后开始向前走并抓取;动作尚未结束时收到下一条指令。"
32
+ },
33
+ {
34
+ "turn_index": 2,
35
+ "source_motion_filename": "wave_two_hands_R_001__A533",
36
+ "audio_path": "/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/qwen3-tts/seed_audio/000155/female_01_zh.wav",
37
+ "spoken_command": "双手在身前挥动",
38
+ "ideal_motion": "新语音刚出现时短暂延续旧动作,听清后平滑切入 listen,再从当时真实姿态开始挥手。"
39
+ },
40
+ {
41
+ "turn_index": 3,
42
+ "source_motion_filename": "neutral_walk_ff_270_R_001__A534_M",
43
+ "audio_path": "/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/qwen3-tts/seed_audio/000120/female_01_zh.wav",
44
+ "spoken_command": "向左走路",
45
+ "ideal_motion": "挥手尚未自然结束时响应新语音,随后相对当前身体朝向向左行走并最终恢复站立。"
46
+ }
47
+ ],
48
+ "ideal_control_timeline": [
49
+ "初始背景输出 </idle>,第一条语音出现后进入 </listen>",
50
+ "每次新语音刚开始时仍执行上一轮 </motion>,不提前读取未来语义",
51
+ "听到有效语音后由 </motion> 平滑切到 </listen>",
52
+ "语音结束后进入新的 </motion>,最后回到 </idle>"
53
+ ],
54
+ "continuity_requirements": [
55
+ "三轮共用一个 Qwen KV cache 和一段连续办公室背景",
56
+ "打断不清空机器人状态或 RTC 历史",
57
+ "新动作从旧动作被打断时的真实位置、朝向和关节状态接续",
58
+ "旧动作的 future progress 仍以其原始完整时长为分母"
59
+ ]
60
+ }
scripts/infer/multiturn_auto_scenario_pause_train_v1.json ADDED
@@ -0,0 +1,41 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "format": "hfd_multiturn_auto_infer_scenario_v1",
3
+ "name": "pause_grab_then_wave_zh",
4
+ "interaction_mode": "pause",
5
+ "interaction_subtype": "silent-pause",
6
+ "data_split": "train",
7
+ "split_seed": 42,
8
+ "val_ratio": 0.1,
9
+ "split_source_namespace": "seed",
10
+ "description": "训练集中文指令在句中静默 1.6 秒;停顿期间继续 listen,完整意图结束后才抓取,随后正常接收第二轮指令。",
11
+ "required_semantics_version": 23,
12
+ "initial_idle_steps": 5,
13
+ "initial_idle_seconds": 2.0,
14
+ "motion_frames_per_turn": 450,
15
+ "motion_seconds_per_turn": 9.0,
16
+ "inter_turn_background_steps_by_turn": [25],
17
+ "inter_turn_background_seconds_by_turn": [10.0],
18
+ "terminal_background_steps": 5,
19
+ "terminal_background_seconds": 2.0,
20
+ "disfluency_event": {"turn_index": 0, "subtype": "silent-pause", "gap_steps": 4, "gap_seconds": 1.6},
21
+ "turns": [
22
+ {
23
+ "turn_index": 1,
24
+ "disfluency_subtype": "silent-pause",
25
+ "source_motion_filename": "grab_walk_ff_180_001__A548",
26
+ "audio_path": "/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/videogen/hfd_hesitation_pause_v1/audio/review_examples/train/pause_grab_zh_female_01_zh.wav",
27
+ "spoken_command": "向前……(静默思考 1.6 秒)……走并抓取某物",
28
+ "final_instruction": "向前走并抓取某物",
29
+ "ideal_motion": "静默段全程保持自然聆听,不提前走;听到完整后向前走并抓取。"
30
+ },
31
+ {
32
+ "turn_index": 2,
33
+ "source_motion_filename": "wave_two_hands_R_001__A533",
34
+ "audio_path": "/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/qwen3-tts/seed_audio/000155/female_01_zh.wav",
35
+ "spoken_command": "双手在身前挥动",
36
+ "ideal_motion": "从第一轮结束状态自然进入第二轮双手挥动。"
37
+ }
38
+ ],
39
+ "ideal_control_timeline": ["背景为 idle", "语音开始后进入 listen", "句中静默仍保持 listen", "完整指令结束后进入 motion", "下一条完整指令按普通 turn-taking 执行"],
40
+ "continuity_requirements": ["整段 session 只使用一个 Qwen KV", "同一连续办公室背景", "RTC、位置和朝向不在停顿或轮次边界重置"]
41
+ }
scripts/infer/multiturn_auto_scenario_pause_train_v2.json ADDED
@@ -0,0 +1,60 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "format": "hfd_multiturn_auto_infer_scenario_v1",
3
+ "name": "pause_grab_then_wave_zh",
4
+ "interaction_mode": "pause",
5
+ "interaction_subtype": "silent-pause",
6
+ "data_split": "train",
7
+ "split_seed": 42,
8
+ "val_ratio": 0.1,
9
+ "split_source_namespace": "seed",
10
+ "description": "训练集中文指令在句中静默 1.6 秒;停顿期间继续 listen,完整意图结束后才抓取,随后正常接收第二轮指令。",
11
+ "required_semantics_version": 24,
12
+ "initial_idle_steps": 5,
13
+ "initial_idle_seconds": 2.0,
14
+ "motion_frames_per_turn": 450,
15
+ "motion_seconds_per_turn": 9.0,
16
+ "inter_turn_background_steps_by_turn": [
17
+ 25
18
+ ],
19
+ "inter_turn_background_seconds_by_turn": [
20
+ 10.0
21
+ ],
22
+ "terminal_background_steps": 5,
23
+ "terminal_background_seconds": 2.0,
24
+ "disfluency_event": {
25
+ "turn_index": 0,
26
+ "subtype": "silent-pause",
27
+ "gap_steps": 4,
28
+ "gap_seconds": 1.6
29
+ },
30
+ "turns": [
31
+ {
32
+ "turn_index": 1,
33
+ "disfluency_subtype": "silent-pause",
34
+ "source_motion_filename": "grab_walk_ff_180_001__A548",
35
+ "audio_path": "/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/videogen/hfd_hesitation_pause_v2/audio/review_examples/train/pause_grab_zh_female_01_zh.wav",
36
+ "spoken_command": "向前……(静默思考 1.6 秒)……走并抓取某物",
37
+ "final_instruction": "向前走并抓取某物",
38
+ "ideal_motion": "静默段全程保持自然聆听,不提前走;听到完整后向前走并抓取。"
39
+ },
40
+ {
41
+ "turn_index": 2,
42
+ "source_motion_filename": "wave_two_hands_R_001__A533",
43
+ "audio_path": "/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/qwen3-tts/seed_audio/000155/female_01_zh.wav",
44
+ "spoken_command": "双手在身前挥动",
45
+ "ideal_motion": "从第一轮结束状态自然进入第二轮双手挥动。"
46
+ }
47
+ ],
48
+ "ideal_control_timeline": [
49
+ "背景为 idle",
50
+ "语音开始后进入 listen",
51
+ "句中静默仍保持 listen",
52
+ "完整指令结束后进入 motion",
53
+ "下一条完整指令按普通 turn-taking 执行"
54
+ ],
55
+ "continuity_requirements": [
56
+ "整段 session 只使用一个 Qwen KV",
57
+ "同一连续办公室背景",
58
+ "RTC、位置和朝向不在停顿或轮次边界重置"
59
+ ]
60
+ }
scripts/infer/multiturn_auto_scenario_safe_hold_val_v1.json ADDED
@@ -0,0 +1,40 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "format": "hfd_multiturn_auto_infer_scenario_v1",
3
+ "name": "hold_val_jump_en",
4
+ "interaction_mode": "safe-hold",
5
+ "data_split": "val",
6
+ "split_seed": 42,
7
+ "val_ratio": 0.1,
8
+ "split_source_namespace": "seed",
9
+ "description": "未见过的跳跃 motion 中收到保持不动,检查是否先落地稳定再保持。",
10
+ "required_semantics_version": 21,
11
+ "initial_idle_steps": 4,
12
+ "initial_idle_seconds": 1.6,
13
+ "motion_frames_per_turn": 450,
14
+ "motion_seconds_per_turn": 9.0,
15
+ "inter_turn_background_steps_by_turn": [7],
16
+ "inter_turn_background_seconds_by_turn": [2.8],
17
+ "terminal_background_steps": 6,
18
+ "terminal_background_seconds": 2.4,
19
+ "turns": [
20
+ {
21
+ "turn_index": 1,
22
+ "source_motion_filename": "reach_jump_R_003__A148_M",
23
+ "audio_path": "/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/qwen3-tts/seed_audio/000243/female_02_en.wav",
24
+ "spoken_command": "reaching something during jump",
25
+ "ideal_motion": "起跳并在空中伸手。"
26
+ },
27
+ {
28
+ "turn_index": 2,
29
+ "control_intent": "safe-hold",
30
+ "audio_path": "/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/videogen/hfd_control_commands_v1/audio/safe-hold/hold_06/female_02_en.wav",
31
+ "spoken_command": "Stay balanced and don't move.",
32
+ "ideal_motion": "如果正在腾空或落地中则先完成安全落地,然后保持真实稳定姿态,不回零到预设 idle。"
33
+ }
34
+ ],
35
+ "ideal_control_timeline": [
36
+ "idle -> listen -> motion",
37
+ "hold 语音在 motion 中到来,没有 listen",
38
+ "不稳定时保持 motion,稳定后转 idle 并保持当时姿"
39
+ ]
40
+ }
scripts/infer/multiturn_auto_scenario_v1.json ADDED
@@ -0,0 +1,61 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "format": "hfd_multiturn_auto_infer_scenario_v1",
3
+ "name": "grab_wave_walk_left_zh",
4
+ "interaction_mode": "turn-taking",
5
+ "data_split": "train",
6
+ "split_seed": 42,
7
+ "val_ratio": 0.1,
8
+ "split_source_namespace": "seed",
9
+ "description": "三轮连续中文指令;同一机器人状态、办公室背景、RTC 历史和 Qwen KV 从头保留到尾。",
10
+ "required_semantics_version": 20,
11
+ "initial_idle_steps": 5,
12
+ "initial_idle_seconds": 2.0,
13
+ "motion_frames_per_turn": 450,
14
+ "motion_seconds_per_turn": 9.0,
15
+ "inter_turn_background_steps_by_turn": [
16
+ 25,
17
+ 30
18
+ ],
19
+ "inter_turn_background_seconds_by_turn": [
20
+ 10.0,
21
+ 12.0
22
+ ],
23
+ "terminal_background_steps": 5,
24
+ "terminal_background_seconds": 2.0,
25
+ "turns": [
26
+ {
27
+ "turn_index": 1,
28
+ "source_motion_filename": "grab_walk_ff_180_001__A548",
29
+ "audio_path": "/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/qwen3-tts/seed_audio/000006/female_01_zh.wav",
30
+ "spoken_command": "向前走并抓取某物",
31
+ "ideal_motion": "听完后向当前身体前方行走,同时完成清晰的伸手抓取;动作结束后在新位置恢复正常站立。"
32
+ },
33
+ {
34
+ "turn_index": 2,
35
+ "source_motion_filename": "wave_two_hands_R_001__A533",
36
+ "audio_path": "/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/qwen3-tts/seed_audio/000155/female_01_zh.wav",
37
+ "spoken_command": "双手在身前挥动",
38
+ "ideal_motion": "保持第一轮累计的位置和朝向,身体基本原地站稳,双手在身前挥动,然后自然收手恢复站立。"
39
+ },
40
+ {
41
+ "turn_index": 3,
42
+ "source_motion_filename": "neutral_walk_ff_270_R_001__A534_M",
43
+ "audio_path": "/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/qwen3-tts/seed_audio/000120/female_01_zh.wav",
44
+ "spoken_command": "向左走路",
45
+ "ideal_motion": "相对第二轮结束时的身体朝向向左侧行走,不重置到世界原点,最后恢复正常站立。"
46
+ }
47
+ ],
48
+ "ideal_control_timeline": [
49
+ "初始纯背景时输出 </idle>,机器人自然站立且不提前抬手或移动",
50
+ "每次有人说话时切到 </listen>,保持自然聆听动作",
51
+ "每条指令结束后切到 </motion>,完整执行该轮动作",
52
+ "第二轮前允许短暂停顿,第三轮前包含约 2 秒额外 idle",
53
+ "最后一轮结束后,在 2 秒纯背景中回到并持续输出 </idle>"
54
+ ],
55
+ "continuity_requirements": [
56
+ "三轮只使用一个 Qwen KV cache,不在轮次边界清空",
57
+ "三轮使用同一段连续办公室背景,不拼接不同背景音",
58
+ "后续动作从上一轮真实生成的最终位置、朝向和关节状态继续",
59
+ "RTC 历史按真实执行的动作连续传递,不读取 GT,也不重置机器人"
60
+ ]
61
+ }
scripts/infer/multiturn_auto_scenario_val_v1.json ADDED
@@ -0,0 +1,100 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "format": "hfd_multiturn_auto_infer_scenario_v1",
3
+ "name": "val_jog_pick_jump_idle_en_timeline_v2",
4
+ "interaction_mode": "turn-taking",
5
+ "description": "训练时 hash 划分到 val 的第 1199 条四轮 session;四条具体 motion source 均未进入训练集,并严格复现该 session 的真实语音起点。",
6
+ "data_split": "val",
7
+ "split_seed": 42,
8
+ "val_ratio": 0.1,
9
+ "split_source_namespace": "seed",
10
+ "validation_session_index": 1199,
11
+ "required_semantics_version": 20,
12
+ "initial_idle_steps": 4,
13
+ "initial_idle_seconds": 1.6,
14
+ "motion_frames_per_turn": 450,
15
+ "motion_seconds_per_turn": 9.0,
16
+ "inter_turn_background_steps_by_turn": [
17
+ 18,
18
+ 32,
19
+ 14
20
+ ],
21
+ "inter_turn_background_seconds_by_turn": [
22
+ 7.2,
23
+ 12.8,
24
+ 5.6
25
+ ],
26
+ "turn_audio_window_counts": [
27
+ 4,
28
+ 18,
29
+ 5,
30
+ 10
31
+ ],
32
+ "expected_turn_window_starts": [
33
+ 0,
34
+ 22,
35
+ 72,
36
+ 91
37
+ ],
38
+ "source_turn_speech_start_frames_50fps": [
39
+ 80,
40
+ 520,
41
+ 1520,
42
+ 1900
43
+ ],
44
+ "source_turn_listen_start_frames_50fps": [
45
+ 120,
46
+ 560,
47
+ 1560,
48
+ 1940
49
+ ],
50
+ "source_inter_turn_idle_steps": [
51
+ 0,
52
+ 8,
53
+ 0
54
+ ],
55
+ "terminal_background_steps": 6,
56
+ "terminal_background_seconds": 2.4,
57
+ "turns": [
58
+ {
59
+ "turn_index": 1,
60
+ "source_motion_filename": "jog_ff_start_180_R_003__A068",
61
+ "audio_path": "/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/qwen3-tts/seed_audio/000175/female_02_en.wav",
62
+ "spoken_command": "start jog forward",
63
+ "ideal_motion": "听完英文指令后从当前站立状态开始向身体前方慢跑,保持朝向连续。"
64
+ },
65
+ {
66
+ "turn_index": 2,
67
+ "source_motion_filename": "medium_big_light_two_hands_right_side_low_to_right_side_high_R_001__A529",
68
+ "audio_path": "/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/qwen3-tts/seed_audio/000227/female_02_en.wav",
69
+ "spoken_command": "picks up medium big light object with two hands from low right side and puts it away to the right side high",
70
+ "ideal_motion": "在第一轮最终位置和朝向上,用双手从右下方拿起物体并举到右上方;这条完整指令和音频目录训练时从未出现。"
71
+ },
72
+ {
73
+ "turn_index": 3,
74
+ "source_motion_filename": "reach_jump_R_003__A148_M",
75
+ "audio_path": "/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/qwen3-tts/seed_audio/000243/female_02_en.wav",
76
+ "spoken_command": "reaching something during jump",
77
+ "ideal_motion": "保持累计状态,完成带伸手够物意图的跳跃并稳定落地。"
78
+ },
79
+ {
80
+ "turn_index": 4,
81
+ "source_motion_filename": "idle_right_to_idle_003__A094_M",
82
+ "audio_path": "/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/qwen3-tts/seed_audio/003713/female_02_en.wav",
83
+ "spoken_command": "from neutral position with left leg forward to neutral position",
84
+ "ideal_motion": "从左腿略在前的自然站姿平稳回到中立站姿,随后在纯背景中回到 idle。"
85
+ }
86
+ ],
87
+ "ideal_control_timeline": [
88
+ "初始纯背景输出 </idle>",
89
+ "每轮语音出现后切换到 </listen>",
90
+ "听清完整指令后切换到 </motion>",
91
+ "四轮语音起点严格复现验证 source 的 0、8.8、28.8、36.4 秒相对时间;7.2、12.8、5.6 秒是扣除上一条语音窗口后的实际背景间隔",
92
+ "第四轮结束后在 2.4 秒背景中回到 </idle>"
93
+ ],
94
+ "continuity_requirements": [
95
+ "四轮共用一个 Qwen KV cache",
96
+ "四轮使用同一段连续办公室背景",
97
+ "机器人位置、朝向、关节状态和 RTC 历史不在轮次间重置",
98
+ "所有 source_motion_filename 均按训练 run 的 split_seed=42、val_ratio=0.1 落在 val"
99
+ ]
100
+ }
scripts/infer/multiturn_auto_suite_v2.json ADDED
@@ -0,0 +1,92 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "format": "hfd_multiturn_auto_infer_suite_v2",
3
+ "description": "同一 checkpoint 的既有五类任务及 pause、filled hesitation、self-repair hesitation × train/val 共十六组闭环推理;由空闲 GPU 动态领取任务。",
4
+ "cases": [
5
+ {
6
+ "interaction_mode": "turn-taking",
7
+ "data_split": "train",
8
+ "scenario_file": "multiturn_auto_scenario_v1.json"
9
+ },
10
+ {
11
+ "interaction_mode": "turn-taking",
12
+ "data_split": "val",
13
+ "scenario_file": "multiturn_auto_scenario_val_v1.json"
14
+ },
15
+ {
16
+ "interaction_mode": "interruption",
17
+ "data_split": "train",
18
+ "scenario_file": "multiturn_auto_scenario_interruption_train_v1.json"
19
+ },
20
+ {
21
+ "interaction_mode": "interruption",
22
+ "data_split": "val",
23
+ "scenario_file": "multiturn_auto_scenario_interruption_val_v1.json"
24
+ },
25
+ {
26
+ "interaction_mode": "emergency-stop",
27
+ "data_split": "train",
28
+ "scenario_file": "multiturn_auto_scenario_emergency_stop_train_v1.json"
29
+ },
30
+ {
31
+ "interaction_mode": "emergency-stop",
32
+ "data_split": "val",
33
+ "scenario_file": "multiturn_auto_scenario_emergency_stop_val_v1.json"
34
+ },
35
+ {
36
+ "interaction_mode": "safe-hold",
37
+ "data_split": "train",
38
+ "scenario_file": "multiturn_auto_scenario_safe_hold_train_v1.json"
39
+ },
40
+ {
41
+ "interaction_mode": "safe-hold",
42
+ "data_split": "val",
43
+ "scenario_file": "multiturn_auto_scenario_safe_hold_val_v1.json"
44
+ },
45
+ {
46
+ "interaction_mode": "backchannel",
47
+ "data_split": "train",
48
+ "scenario_file": "multiturn_auto_scenario_backchannel_train_v1.json"
49
+ },
50
+ {
51
+ "interaction_mode": "backchannel",
52
+ "data_split": "val",
53
+ "scenario_file": "multiturn_auto_scenario_backchannel_val_v1.json"
54
+ },
55
+ {
56
+ "interaction_mode": "pause",
57
+ "interaction_subtype": "silent-pause",
58
+ "data_split": "train",
59
+ "scenario_file": "multiturn_auto_scenario_pause_train_v1.json"
60
+ },
61
+ {
62
+ "interaction_mode": "pause",
63
+ "interaction_subtype": "silent-pause",
64
+ "data_split": "val",
65
+ "scenario_file": "multiturn_auto_scenario_pause_val_v1.json"
66
+ },
67
+ {
68
+ "interaction_mode": "hesitation",
69
+ "interaction_subtype": "filled-pause",
70
+ "data_split": "train",
71
+ "scenario_file": "multiturn_auto_scenario_hesitation_filled_train_v1.json"
72
+ },
73
+ {
74
+ "interaction_mode": "hesitation",
75
+ "interaction_subtype": "filled-pause",
76
+ "data_split": "val",
77
+ "scenario_file": "multiturn_auto_scenario_hesitation_filled_val_v1.json"
78
+ },
79
+ {
80
+ "interaction_mode": "hesitation",
81
+ "interaction_subtype": "self-repair",
82
+ "data_split": "train",
83
+ "scenario_file": "multiturn_auto_scenario_hesitation_repair_train_v1.json"
84
+ },
85
+ {
86
+ "interaction_mode": "hesitation",
87
+ "interaction_subtype": "self-repair",
88
+ "data_split": "val",
89
+ "scenario_file": "multiturn_auto_scenario_hesitation_repair_val_v1.json"
90
+ }
91
+ ]
92
+ }
scripts/infer/multiturn_auto_suite_v3.json ADDED
@@ -0,0 +1,92 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "format": "hfd_multiturn_auto_infer_suite_v3",
3
+ "description": "同一 checkpoint 的七类任务 × train/val 共十六组闭环推理;v24 使用自然拖长 filler,并同时覆盖显式提示和无提示词自我改口;由空闲 GPU 动态领取任务。",
4
+ "cases": [
5
+ {
6
+ "interaction_mode": "turn-taking",
7
+ "data_split": "train",
8
+ "scenario_file": "multiturn_auto_scenario_v1.json"
9
+ },
10
+ {
11
+ "interaction_mode": "turn-taking",
12
+ "data_split": "val",
13
+ "scenario_file": "multiturn_auto_scenario_val_v1.json"
14
+ },
15
+ {
16
+ "interaction_mode": "interruption",
17
+ "data_split": "train",
18
+ "scenario_file": "multiturn_auto_scenario_interruption_train_v1.json"
19
+ },
20
+ {
21
+ "interaction_mode": "interruption",
22
+ "data_split": "val",
23
+ "scenario_file": "multiturn_auto_scenario_interruption_val_v1.json"
24
+ },
25
+ {
26
+ "interaction_mode": "emergency-stop",
27
+ "data_split": "train",
28
+ "scenario_file": "multiturn_auto_scenario_emergency_stop_train_v1.json"
29
+ },
30
+ {
31
+ "interaction_mode": "emergency-stop",
32
+ "data_split": "val",
33
+ "scenario_file": "multiturn_auto_scenario_emergency_stop_val_v1.json"
34
+ },
35
+ {
36
+ "interaction_mode": "safe-hold",
37
+ "data_split": "train",
38
+ "scenario_file": "multiturn_auto_scenario_safe_hold_train_v1.json"
39
+ },
40
+ {
41
+ "interaction_mode": "safe-hold",
42
+ "data_split": "val",
43
+ "scenario_file": "multiturn_auto_scenario_safe_hold_val_v1.json"
44
+ },
45
+ {
46
+ "interaction_mode": "backchannel",
47
+ "data_split": "train",
48
+ "scenario_file": "multiturn_auto_scenario_backchannel_train_v1.json"
49
+ },
50
+ {
51
+ "interaction_mode": "backchannel",
52
+ "data_split": "val",
53
+ "scenario_file": "multiturn_auto_scenario_backchannel_val_v1.json"
54
+ },
55
+ {
56
+ "interaction_mode": "pause",
57
+ "interaction_subtype": "silent-pause",
58
+ "data_split": "train",
59
+ "scenario_file": "multiturn_auto_scenario_pause_train_v2.json"
60
+ },
61
+ {
62
+ "interaction_mode": "pause",
63
+ "interaction_subtype": "silent-pause",
64
+ "data_split": "val",
65
+ "scenario_file": "multiturn_auto_scenario_pause_val_v2.json"
66
+ },
67
+ {
68
+ "interaction_mode": "hesitation",
69
+ "interaction_subtype": "filled-pause",
70
+ "data_split": "train",
71
+ "scenario_file": "multiturn_auto_scenario_hesitation_filled_train_v2.json"
72
+ },
73
+ {
74
+ "interaction_mode": "hesitation",
75
+ "interaction_subtype": "filled-pause",
76
+ "data_split": "val",
77
+ "scenario_file": "multiturn_auto_scenario_hesitation_filled_val_v2.json"
78
+ },
79
+ {
80
+ "interaction_mode": "hesitation",
81
+ "interaction_subtype": "self-repair",
82
+ "data_split": "train",
83
+ "scenario_file": "multiturn_auto_scenario_hesitation_repair_train_v2.json"
84
+ },
85
+ {
86
+ "interaction_mode": "hesitation",
87
+ "interaction_subtype": "self-repair",
88
+ "data_split": "val",
89
+ "scenario_file": "multiturn_auto_scenario_hesitation_repair_val_v2.json"
90
+ }
91
+ ]
92
+ }
scripts/infer/mux_full_duplex_audio.py ADDED
@@ -0,0 +1,180 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/usr/bin/env python3
2
+ from __future__ import annotations
3
+
4
+ import argparse
5
+ import json
6
+ import subprocess
7
+ from pathlib import Path
8
+ from typing import Any
9
+
10
+
11
+ def probe(path: Path) -> dict[str, Any]:
12
+ result = subprocess.run(
13
+ [
14
+ "ffprobe",
15
+ "-v",
16
+ "error",
17
+ "-show_streams",
18
+ "-show_format",
19
+ "-of",
20
+ "json",
21
+ str(path),
22
+ ],
23
+ capture_output=True,
24
+ check=False,
25
+ )
26
+ if result.returncode != 0:
27
+ raise RuntimeError(result.stderr.decode("utf-8", errors="ignore"))
28
+ return json.loads(result.stdout)
29
+
30
+
31
+ def stream(payload: dict[str, Any], codec_type: str) -> dict[str, Any]:
32
+ for item in payload.get("streams", []):
33
+ if item.get("codec_type") == codec_type:
34
+ return item
35
+ raise RuntimeError(f"ffprobe found no {codec_type} stream")
36
+
37
+
38
+ def fraction(value: str) -> float:
39
+ numerator, denominator = str(value).split("/", 1)
40
+ return float(numerator) / float(denominator)
41
+
42
+
43
+ def main() -> None:
44
+ parser = argparse.ArgumentParser(
45
+ description=(
46
+ "Mux either a complete HFD streaming audio timeline or legacy "
47
+ "speech-only audio onto the rendered motion"
48
+ )
49
+ )
50
+ parser.add_argument("--video", required=True)
51
+ parser.add_argument("--audio", required=True)
52
+ parser.add_argument("--output", required=True)
53
+ parser.add_argument("--infer-meta", required=True)
54
+ parser.add_argument(
55
+ "--audio-is-timeline",
56
+ action="store_true",
57
+ help=(
58
+ "The audio already contains initial background, speech, and "
59
+ "post-WAV background on the complete 50 Hz output timeline."
60
+ ),
61
+ )
62
+ parser.add_argument("--expected-fps", type=float, default=50.0)
63
+ args = parser.parse_args()
64
+
65
+ video = Path(args.video).expanduser().resolve()
66
+ audio = Path(args.audio).expanduser().resolve()
67
+ output = Path(args.output).expanduser().resolve()
68
+ meta_path = Path(args.infer_meta).expanduser().resolve()
69
+ for path in (video, audio, meta_path):
70
+ if not path.is_file():
71
+ raise FileNotFoundError(path)
72
+
73
+ video_probe = probe(video)
74
+ video_stream = stream(video_probe, "video")
75
+ video_duration = float(
76
+ video_stream.get("duration") or video_probe["format"].get("duration")
77
+ )
78
+ video_fps = fraction(video_stream.get("avg_frame_rate") or video_stream["r_frame_rate"])
79
+ if abs(video_fps - float(args.expected_fps)) > 0.01:
80
+ raise RuntimeError(
81
+ f"Rendered video FPS {video_fps} does not preserve the 50 Hz HFD timeline"
82
+ )
83
+
84
+ metadata = json.loads(meta_path.read_text(encoding="utf-8"))
85
+ requested_speech_delay_seconds = max(
86
+ float(metadata.get("speech_start_seconds", 0.0)),
87
+ 0.0,
88
+ )
89
+ applied_audio_delay_seconds = (
90
+ 0.0 if bool(args.audio_is_timeline) else requested_speech_delay_seconds
91
+ )
92
+ speech_delay_ms = int(round(applied_audio_delay_seconds * 1000.0))
93
+ delay_filter = (
94
+ f"adelay={speech_delay_ms}:all=1,"
95
+ if speech_delay_ms > 0
96
+ else ""
97
+ )
98
+ output.parent.mkdir(parents=True, exist_ok=True)
99
+ temporary = output.with_name(f".{output.name}.tmp.mp4")
100
+ command = [
101
+ "ffmpeg",
102
+ "-v",
103
+ "error",
104
+ "-y",
105
+ "-i",
106
+ str(video),
107
+ "-i",
108
+ str(audio),
109
+ "-filter_complex",
110
+ f"[1:a]{delay_filter}aresample=async=1:first_pts=0,"
111
+ f"apad=whole_dur={video_duration:.9f}[audio]",
112
+ "-map",
113
+ "0:v:0",
114
+ "-map",
115
+ "[audio]",
116
+ "-c:v",
117
+ "copy",
118
+ "-c:a",
119
+ "aac",
120
+ "-movflags",
121
+ "+faststart",
122
+ "-t",
123
+ f"{video_duration:.9f}",
124
+ str(temporary),
125
+ ]
126
+ result = subprocess.run(command, capture_output=True, check=False)
127
+ if result.returncode != 0:
128
+ raise RuntimeError(
129
+ f"ffmpeg mux failed: {result.stderr.decode('utf-8', errors='ignore')}"
130
+ )
131
+ temporary.replace(output)
132
+
133
+ output_probe = probe(output)
134
+ output_video = stream(output_probe, "video")
135
+ output_audio = stream(output_probe, "audio")
136
+ video_start = float(output_video.get("start_time", 0.0))
137
+ audio_start = float(output_audio.get("start_time", 0.0))
138
+ output_duration = float(output_probe["format"].get("duration", 0.0))
139
+ frame_tolerance = 1.0 / float(args.expected_fps) + 1e-3
140
+ if abs(video_start) > frame_tolerance or abs(audio_start) > frame_tolerance:
141
+ raise RuntimeError(
142
+ f"Muxed streams do not start together: video={video_start}, audio={audio_start}"
143
+ )
144
+ if output_duration + frame_tolerance < video_duration:
145
+ raise RuntimeError(
146
+ f"Mux truncated the motion video: input={video_duration}, output={output_duration}"
147
+ )
148
+
149
+ metadata["mux"] = {
150
+ "video_without_audio": str(video),
151
+ "video_with_audio": str(output),
152
+ "audio": str(audio),
153
+ "video_fps": video_fps,
154
+ "video_start_seconds": video_start,
155
+ "audio_start_seconds": audio_start,
156
+ "video_duration_seconds": video_duration,
157
+ "muxed_duration_seconds": output_duration,
158
+ "audio_mode": (
159
+ "complete_stream_timeline"
160
+ if bool(args.audio_is_timeline)
161
+ else "speech_only_legacy"
162
+ ),
163
+ "audio_padding": (
164
+ "timeline_apad_only_if_container_rounding_requires_it"
165
+ if bool(args.audio_is_timeline)
166
+ else "silence_to_video_end"
167
+ ),
168
+ "speech_delay_seconds": requested_speech_delay_seconds,
169
+ "applied_audio_delay_seconds": applied_audio_delay_seconds,
170
+ "motion_start_frame_50fps": metadata.get("motion_start_frame_50fps"),
171
+ "motion_start_seconds": metadata.get("motion_start_seconds"),
172
+ }
173
+ meta_path.write_text(
174
+ json.dumps(metadata, ensure_ascii=False, indent=2) + "\n", encoding="utf-8"
175
+ )
176
+ print(f"[done] synchronized video: {output}", flush=True)
177
+
178
+
179
+ if __name__ == "__main__":
180
+ main()
scripts/infer/render_third_party_review.py ADDED
@@ -0,0 +1,287 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/usr/bin/env python3
2
+ """Render a stratified final-model review from saved predictions, CPU only."""
3
+ from __future__ import annotations
4
+
5
+ import argparse
6
+ from concurrent.futures import ThreadPoolExecutor, as_completed
7
+ import hashlib
8
+ import json
9
+ import os
10
+ from pathlib import Path
11
+ import random
12
+ import shutil
13
+ import subprocess
14
+ import sys
15
+ import time
16
+ import wave
17
+
18
+ PROJECT = Path(__file__).resolve().parents[2]
19
+ sys.path[:0] = [str(PROJECT), str(PROJECT / "src")]
20
+ PROFILES = ("single_short", "multi_single", "multi_babble", "mixed_short", "continuous", "same_voice_aside")
21
+ LABELS = dict(zip(PROFILES, ("单段短人声", "多段单人声", "多段多人杂谈", "单/多人混合短声", "连续背景说话", "指令同音色旁话")))
22
+ ASSETS = ("motion.npy", "body_pos.csv", "body_quat.csv", "joint_pos.csv", "body_lin_vel.csv",
23
+ "body_ang_vel.csv", "joint_vel.csv", "metadata.txt", "infer_meta.json", "metrics.json", "session_recipe.json")
24
+
25
+
26
+ def read(path):
27
+ return json.loads(Path(path).read_text())
28
+
29
+
30
+ def digest(path):
31
+ h = hashlib.sha256()
32
+ with Path(path).open("rb") as handle:
33
+ for block in iter(lambda: handle.read(1024 * 1024), b""):
34
+ h.update(block)
35
+ return h.hexdigest()
36
+
37
+
38
+ def write(path, value):
39
+ path = Path(path)
40
+ temp = path.with_suffix(path.suffix + ".tmp")
41
+ temp.write_text(json.dumps(value, ensure_ascii=False, indent=2, allow_nan=False) + "\n")
42
+ temp.replace(path)
43
+
44
+
45
+ def select_samples(selection, report, seed=26100801):
46
+ rows, samples = selection["sessions"], report["samples"]
47
+ ids = [r["session_id"] for r in rows]
48
+ sample_ids = [r["session_id"] for r in samples]
49
+ if (report.get("status") != "complete" or report.get("evaluation_split") != "train_only"
50
+ or report.get("count") != 300 or len(rows) != 300 or len(samples) != 300
51
+ or len(set(ids)) != 300 or len(set(sample_ids)) != 300
52
+ or set(ids) != set(sample_ids) or selection["session_ids"] != ids):
53
+ raise ValueError("Expected the same complete unique train300 selection and report")
54
+ by_id = {r["session_id"]: r for r in rows}
55
+ for sample in samples:
56
+ row = by_id[sample["session_id"]]
57
+ if (sample["profile"] != row["session_type"]
58
+ or sample["background_language"] != row["background_language"]):
59
+ raise ValueError("Prediction profile/language differs from frozen selection")
60
+ rng, chosen = random.Random(seed), []
61
+ for profile in PROFILES:
62
+ for language in ("zh", "en"):
63
+ for high in (False, True):
64
+ pool = sorted((s for s in samples if s["profile"] == profile
65
+ and s["background_language"] == language
66
+ and by_id[s["session_id"]]["background_high"] == high), key=lambda s: s["session_id"])
67
+ if not pool:
68
+ raise ValueError(f"Empty review cell: {profile}/{language}/{high}")
69
+ chosen.append({**rng.choice(pool), "number": f"{len(chosen) + 1:02d}", "background_high": high})
70
+ return chosen
71
+
72
+
73
+ def contained(path, root):
74
+ path, root = Path(path).resolve(strict=True), Path(root).resolve(strict=True)
75
+ if path == root or not path.is_relative_to(root):
76
+ raise ValueError(f"Path escaped expected input root: {path}")
77
+ return path
78
+
79
+
80
+ def render_session_alive(pid):
81
+ """Include orphaned encoders/renderers, not only the coordinating worker."""
82
+ for stat in Path("/proc").glob("[0-9]*/stat"):
83
+ try:
84
+ fields = stat.read_text().rsplit(")", 1)[1].split()
85
+ except (FileNotFoundError, ProcessLookupError):
86
+ continue
87
+ if fields[0] != "Z" and int(fields[3]) == int(pid):
88
+ return True
89
+ return False
90
+
91
+
92
+ def prepare(run, manifest):
93
+ import numpy as np
94
+ run = contained(run, PROJECT / "runs")
95
+ report_path = run / "final_train300/report.json"
96
+ report, result, selection = read(report_path), read(run / "result.json"), read(manifest)
97
+ if result["status"] != "complete" or report["step"] != result["step"]:
98
+ raise ValueError("Final report does not describe the completed run")
99
+ status = read(run / "run_status.json")
100
+ if digest(manifest) != status["manifest_sha256"]:
101
+ raise ValueError("Frozen train300 selection hash changed")
102
+ chosen = select_samples(selection, report)
103
+ for row in chosen:
104
+ source = contained(row["prediction_directory"], run / "final_train300/samples")
105
+ meta = read(source / "infer_meta.json")
106
+ recipe = read(source / "session_recipe.json")
107
+ if (meta["session_id"] != row["session_id"] or meta["global_step"] != report["step"]
108
+ or meta["artifact_role"] != "model_prediction_not_GT"
109
+ or recipe["split"] != "train" or recipe["session_id"] != row["session_id"]):
110
+ raise ValueError("Prediction provenance is inconsistent")
111
+ audio = contained(meta["frozen_audio_path"], PROJECT / "data/hfd_third_party_multi_v5/production/audio/train")
112
+ if digest(audio) != meta["frozen_audio_sha256"] or meta["frozen_audio_sha256"] != recipe["audio"]["sha256"]:
113
+ raise ValueError("Frozen mixed audio changed")
114
+ with wave.open(str(audio), "rb") as wav:
115
+ if (wav.getframerate(), wav.getnchannels(), wav.getsampwidth()) != (16000, 1, 2):
116
+ raise ValueError("Expected the original mono 16 kHz PCM16")
117
+ frames, seconds = round(wav.getnframes() * 50 / 16000), wav.getnframes() / 16000
118
+ motion = np.load(source / "motion.npy", allow_pickle=False)
119
+ if motion.shape != (frames, 36) or not np.isfinite(motion).all() or digest(source / "motion.npy") != row["motion_sha256"]:
120
+ raise ValueError("Saved predicted trajectory is incomplete or changed")
121
+ for name, width in (("body_pos.csv", 3), ("body_quat.csv", 4), ("joint_pos.csv", 29)):
122
+ values = np.loadtxt(source / name, delimiter=",", skiprows=1, ndmin=2)
123
+ if values.shape != (frames, width) or not np.isfinite(values).all():
124
+ raise ValueError(f"Render CSV has invalid length or values: {name}")
125
+ row.update(source_directory=str(source), audio_path=str(audio), audio_sha256=meta["frozen_audio_sha256"],
126
+ duration_seconds=seconds, source_files={name: digest(source / name) for name in ASSETS})
127
+ return {"schema": 1, "run_dir": str(run), "step": report["step"], "artifact_role": "model_prediction_not_GT",
128
+ "evaluation_split": "train_only", "seed": 26100801, "count": 24, "inference_repeated": False,
129
+ "report_sha256": digest(report_path), "manifest_sha256": digest(manifest),
130
+ "selection_policy": "one random sample per profile/language/loudness cell; never select by score",
131
+ "render": {"backend": "mesa_cpu", "workers": 4, "threads_each": 4, "fps": 25, "width": 320, "height": 240},
132
+ "audio_policy": "original mixed timeline; no gain/offset change; AAC encoding in MP4",
133
+ "samples": chosen}
134
+
135
+
136
+ def render_one(row, output):
137
+ from scripts.infer.third_party_checkpoint_probe import render_prediction
138
+ from scripts.infer.mux_full_duplex_audio import probe, stream, fraction
139
+ started = time.monotonic()
140
+ source, audio = Path(row["source_directory"]), Path(row["audio_path"])
141
+ destination = output / "samples" / row["number"]
142
+ flat = output / f"{row['number']}.mp4"
143
+ saved_result = destination / "render_result.json"
144
+ for path in (output, output / "samples", destination, flat, saved_result, *(destination / name for name in ASSETS)):
145
+ if path.is_symlink():
146
+ raise ValueError("Refusing symlink in dedicated preview outputs")
147
+ if saved_result.exists():
148
+ result = read(saved_result)
149
+ if (result.get("status") != "complete" or any(result.get(key) != value for key, value in row.items())
150
+ or not result.get("full_decode_passed") or not result.get("source_files_unchanged")
151
+ or result.get("video") != str(flat) or digest(flat) != result["video_sha256"]
152
+ or flat.stat().st_size != result.get("video_bytes")):
153
+ raise ValueError("Completed preview does not match the frozen render plan")
154
+ if any(digest(source / name) != expected for name, expected in row["source_files"].items()) or digest(audio) != row["audio_sha256"]:
155
+ raise ValueError("Original prediction/audio changed before preview reuse")
156
+ return {**result, "reused_existing_video": True}
157
+ if flat.exists():
158
+ raise ValueError("Unverified video exists without a completed render result; refusing overwrite")
159
+ # Only unfinished copies in this dedicated output may be rewritten on resume.
160
+ destination.mkdir(parents=True, exist_ok=True)
161
+ for name, expected in row["source_files"].items():
162
+ if digest(source / name) != expected:
163
+ raise ValueError("Source prediction changed after preflight")
164
+ shutil.copy2(source / name, destination / name)
165
+ if digest(destination / name) != expected:
166
+ raise ValueError("Prediction copy changed")
167
+ if digest(audio) != row["audio_sha256"]:
168
+ raise ValueError("Audio changed after preflight")
169
+ video = render_prediction(destination, audio)
170
+ media = probe(video); v, a = stream(media, "video"), stream(media, "audio")
171
+ if (v["width"], v["height"]) != (320, 240) or abs(fraction(v["avg_frame_rate"]) - 25) > .01:
172
+ raise ValueError("Rendered dimensions/FPS changed")
173
+ if int(a["channels"]) != 1 or int(a["sample_rate"]) != 16000:
174
+ raise ValueError("Rendered audio stream format changed")
175
+ if float(media["format"]["duration"]) < row["duration_seconds"] - .041:
176
+ raise ValueError("Rendered video truncated the input timeline")
177
+ for name, expected in row["source_files"].items():
178
+ if digest(source / name) != expected:
179
+ raise ValueError("Original prediction was modified during rendering")
180
+ if digest(audio) != row["audio_sha256"]:
181
+ raise ValueError("Original audio was modified during rendering")
182
+ # A second name for the same local output; no duplicate video storage.
183
+ os.link(video, flat)
184
+ result = {**row, "status": "complete", "video": str(flat), "video_sha256": digest(flat),
185
+ "video_bytes": flat.stat().st_size, "full_decode_passed": True, "source_files_unchanged": True,
186
+ "render_seconds": time.monotonic() - started}
187
+ write(destination / "render_result.json", result)
188
+ return result
189
+
190
+
191
+ def publish(output, plan, completed):
192
+ lines = ["# 最终模型预测视频(不是 GT)", "", f"Checkpoint **{plan['step']}**;已完成 **{len(completed)}/24**。",
193
+ "", "复用已保存的训练集预测,不重复推理;六类×中英文背景×普通/较高背景音量,每格固定随机抽一条。",
194
+ "320×240、25fps审核预览;原始预测仍为50Hz。完整原混音不调音量、不改时间位置,MP4音轨编码为AAC。",
195
+ "", "下表指标对应各自单条样本,仅供定位问题,不作为总体效果估计。", "",
196
+ "| 序号/视频 | 类型 | 背景语言 | 背景音量 | 时长 | 控制准确率 | 背景误动率 |",
197
+ "|---|---|---|---|---:|---:|---:|"]
198
+ by_number = {r["number"]: r for r in completed}
199
+ for row in plan["samples"]:
200
+ number = row["number"]; label = f"[{number}]({number}.mp4)" if number in by_number else f"{number} 待完成"
201
+ metrics = row["metrics"]; bg = metrics.get("background_only_false_motion_rate")
202
+ bg = "无可评分区间" if bg is None else f"{bg:.1%}"
203
+ lines.append(f"| {label} | {LABELS[row['profile']]} | {'中文' if row['background_language']=='zh' else '英文'} | "
204
+ f"{'较高' if row['background_high'] else '普通'} | {row['duration_seconds']:.1f}s | {metrics['control_accuracy']:.1%} | {bg} |")
205
+ lines.extend(["", "[原始逐条评估](../final_train300/report.json) · [抽样与来源校验](plan.json) · [渲染状态](status.json)", ""])
206
+ (output / "README.md").write_text("\n".join(lines))
207
+ run = Path(plan["run_dir"]); review = run / "review.md"
208
+ if review.exists():
209
+ marker = "<!-- final_prediction_videos -->"
210
+ text = review.read_text().split(marker)[0].rstrip()
211
+ text = text.replace("| — | 尚无完整 probe 记录 | — | — | — | — |",
212
+ "| — | 本轮采用轮末数值评估,见下方逐轮表 | — | — | — | [预测视频](prediction_videos/README.md) |")
213
+ review.write_text(text + f"\n\n{marker}\n## 最终模型预测视频\n\n"
214
+ f"Checkpoint {plan['step']},已渲染 {len(completed)}/24 条,**模型预测,不是GT**。\n\n"
215
+ "[01–24 视频与类型表](prediction_videos/README.md) · [最终300条评估](final_train300/report.json)\n")
216
+
217
+
218
+ def worker(output):
219
+ # Imports occur before parallel rendering; no CUDA model is loaded.
220
+ from scripts.infer.third_party_checkpoint_probe import check_renderer
221
+ check_renderer()
222
+ plan = read(output / "plan.json")
223
+ completed, failures = [], []
224
+ started = time.time()
225
+ def status(phase):
226
+ write(output / "status.json", {"phase": phase, "pid": os.getpid(), "started_at": started,
227
+ "updated_at": time.time(), "completed": len(completed), "total": 24, "failed": len(failures),
228
+ "samples": completed, "errors": failures, "gpu_used": False, "inference_repeated": False,
229
+ "workers": plan["render"]["workers"]})
230
+ publish(output, plan, completed)
231
+ status("rendering")
232
+ with ThreadPoolExecutor(max_workers=plan["render"]["workers"]) as pool:
233
+ jobs = {pool.submit(render_one, row, output): row for row in plan["samples"]}
234
+ for future in as_completed(jobs):
235
+ row = jobs[future]
236
+ try:
237
+ result = future.result(); completed.append(result)
238
+ print(json.dumps({"number": row["number"], "seconds": result["render_seconds"], "status": "complete"}), flush=True)
239
+ except Exception as error:
240
+ failures.append({"number": row["number"], "error": repr(error)})
241
+ print(json.dumps(failures[-1]), flush=True)
242
+ status("rendering")
243
+ status("failed" if failures else "completed")
244
+ if failures:
245
+ raise RuntimeError(f"{len(failures)} preview renders failed; inspect per-sample render.log")
246
+
247
+
248
+ def main():
249
+ parser = argparse.ArgumentParser(description=__doc__)
250
+ parser.add_argument("--run-dir", type=Path, required=True)
251
+ parser.add_argument("--manifest", type=Path, default=PROJECT / "data/hfd_third_party_multi_v5/training_pilot_300/selection.json")
252
+ parser.add_argument("--workers", type=int, choices=range(1, 17), default=12)
253
+ parser.add_argument("--resume", action="store_true", help="Reuse verified videos after the previous render worker has exited")
254
+ mode = parser.add_mutually_exclusive_group(required=True)
255
+ for name in ("launch", "worker", "check-only"):
256
+ mode.add_argument(f"--{name}", action="store_true")
257
+ args = parser.parse_args(); run = contained(args.run_dir, PROJECT / "runs"); output = run / "prediction_videos"
258
+ if args.resume and not args.launch:
259
+ parser.error("--resume requires --launch")
260
+ os.environ.update(CUDA_VISIBLE_DEVICES="", PYTHONDONTWRITEBYTECODE="1", OMP_NUM_THREADS="1", MKL_NUM_THREADS="1")
261
+ if args.worker:
262
+ worker(output); return
263
+ plan = prepare(run, args.manifest)
264
+ plan["render"]["workers"] = args.workers
265
+ if args.check_only:
266
+ print(json.dumps({"passed": True, "step": plan["step"], "count": 24, "seconds": sum(r['duration_seconds'] for r in plan['samples'])})); return
267
+ if args.resume:
268
+ previous = read(output / "plan.json")
269
+ if any(previous[key] != plan[key] for key in plan if key != "render"):
270
+ raise ValueError("Cannot resume a different frozen prediction review")
271
+ if any(previous["render"][key] != plan["render"][key] for key in plan["render"] if key != "workers"):
272
+ raise ValueError("Resume can change worker count only, not rendering settings")
273
+ previous_status = read(output / "status.json")
274
+ if render_session_alive(previous_status["pid"]):
275
+ raise ValueError("Previous render session is still present; refusing concurrent launch")
276
+ else:
277
+ output.mkdir(exist_ok=False)
278
+ write(output / "plan.json", plan)
279
+ with (output / "render.log").open("a" if args.resume else "x") as log:
280
+ child = subprocess.Popen([sys.executable, str(Path(__file__).resolve()), "--run-dir", str(run), "--worker"],
281
+ cwd=PROJECT, env=os.environ.copy(), stdin=subprocess.DEVNULL, stdout=log, stderr=subprocess.STDOUT,
282
+ start_new_session=True)
283
+ print(json.dumps({"render_pid": child.pid, "output": str(output), "index": str(output / 'README.md')}))
284
+
285
+
286
+ if __name__ == "__main__":
287
+ main()
scripts/infer/render_third_party_scale.py ADDED
@@ -0,0 +1,151 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/usr/bin/env python3
2
+ """CPU-only final scale previews, with separate train and held-out indexes."""
3
+ from __future__ import annotations
4
+
5
+ import argparse
6
+ from concurrent.futures import ThreadPoolExecutor, as_completed
7
+ import os
8
+ from pathlib import Path
9
+ import sys
10
+ import time
11
+ import wave
12
+
13
+ PROJECT = Path(__file__).resolve().parents[2]
14
+ sys.path[:0] = [str(PROJECT), str(PROJECT / "src")]
15
+ from scripts.infer.render_third_party_review import (
16
+ ASSETS, LABELS, contained, digest, read, render_one, write)
17
+
18
+
19
+ def selected_rows(selection, report, step):
20
+ if (selection.get("scaleup") is not True or selection.get("split") not in ("train", "validation")
21
+ or report.get("status") != "complete" or report.get("step") != step
22
+ or report.get("evaluation_split") != selection["split"]):
23
+ raise ValueError("Final prediction split/step differs from its frozen selection")
24
+ by_id = {r["session_id"]: r for r in selection["sessions"]}
25
+ samples = {r["session_id"]: r for r in report["samples"]}
26
+ if len(samples) != report["count"] or len(samples) != len(report["samples"]):
27
+ raise ValueError("Duplicate/incomplete final predictions")
28
+ ids = selection["video_session_ids"]
29
+ if not ids or len(ids) != len(set(ids)) or not set(ids) <= samples.keys() & by_id.keys():
30
+ raise ValueError("Missing predetermined video samples")
31
+ rows = []
32
+ for index, sid in enumerate(ids, 1):
33
+ row, chosen = samples[sid], by_id[sid]
34
+ if (row["profile"] != chosen["session_type"] or row["split"] != selection["split"]
35
+ or row["background_language"] != chosen["background_language"]):
36
+ raise ValueError("Prediction labels differ from frozen selection")
37
+ rows.append({**row, "number": f"{index:02d}", "background_high": chosen["background_high"]})
38
+ return rows
39
+
40
+
41
+ def prepare(run, manifest):
42
+ import numpy as np
43
+ run = contained(run, PROJECT / "runs")
44
+ result, identity = read(run / "result.json"), read(run / "scale_identity.json")
45
+ if result["status"] != "complete":
46
+ raise ValueError("Training/evaluation has not completed")
47
+ rows, hashes = [], {}
48
+ for split in ("train", "validation"):
49
+ path = manifest if split == "train" else manifest.parent / "validation.json"
50
+ key = "manifest_sha256" if split == "train" else "validation_sha256"
51
+ if digest(path) != identity[key]:
52
+ raise ValueError("Frozen video selection changed")
53
+ selection = read(path)
54
+ report_path = run / "evaluations/final" / split / "report.json"
55
+ report = read(report_path); hashes[split] = digest(report_path)
56
+ for row in selected_rows(selection, report, result["step"]):
57
+ source = contained(row["prediction_directory"], report_path.parent / "samples")
58
+ meta, recipe = read(source / "infer_meta.json"), read(source / "session_recipe.json")
59
+ if (meta["session_id"] != row["session_id"] or meta["global_step"] != result["step"]
60
+ or meta.get("evaluation_split") != split or meta["artifact_role"] != "model_prediction_not_GT"
61
+ or recipe["session_id"] != row["session_id"] or recipe["split"] != split):
62
+ raise ValueError("Prediction provenance mismatch")
63
+ audio = contained(meta["frozen_audio_path"], Path(selection["production_root"]) / "audio" / split)
64
+ if digest(audio) != meta["frozen_audio_sha256"] or digest(audio) != recipe["audio"]["sha256"]:
65
+ raise ValueError("Original mixed audio changed")
66
+ with wave.open(str(audio), "rb") as wav:
67
+ if (wav.getframerate(), wav.getnchannels(), wav.getsampwidth()) != (16000, 1, 2):
68
+ raise ValueError("Expected original mono 16kHz PCM16")
69
+ seconds = wav.getnframes() / 16000
70
+ frames = round(seconds * 50)
71
+ motion = np.load(source / "motion.npy", allow_pickle=False)
72
+ if motion.shape != (frames, 36) or not np.isfinite(motion).all() or digest(source / "motion.npy") != row["motion_sha256"]:
73
+ raise ValueError("Prediction trajectory changed or incomplete")
74
+ for name, width in (("body_pos.csv", 3), ("body_quat.csv", 4), ("joint_pos.csv", 29)):
75
+ values = np.loadtxt(source / name, delimiter=",", skiprows=1, ndmin=2)
76
+ if values.shape != (frames, width) or not np.isfinite(values).all():
77
+ raise ValueError(f"Invalid predicted render CSV: {name}")
78
+ rows.append({**row, "source_directory": str(source), "audio_path": str(audio),
79
+ "audio_sha256": meta["frozen_audio_sha256"], "duration_seconds": seconds,
80
+ "source_files": {name: digest(source / name) for name in ASSETS}})
81
+ return {"schema": 1, "run_dir": str(run), "step": result["step"], "count": len(rows), "samples": rows,
82
+ "report_hashes": hashes, "gpu_used": False, "inference_repeated": False,
83
+ "selection_policy": "predetermined one per available class/language/loudness cell, never by model score",
84
+ "artifact_role": "model_prediction_not_GT", "render": {"workers": 12, "fps": 25, "size": [320, 240]}}
85
+
86
+
87
+ def publish(output, plan, completed):
88
+ done = {(r["split"], r["number"]) for r in completed}
89
+ lines = ["# 全量训练最终预测视频(不是 GT)", "",
90
+ f"新阶段 checkpoint {plan['step']};完成 {len(completed)}/{plan['count']}。",
91
+ "复用完整预测;原混音不改变音量和时间。CPU 并行渲染,320×240、25fps;原预测仍为50Hz。",
92
+ "样例在训练前按六类×中英文背景×普通/较高音量固定随机选定,不按效果挑选。", ""]
93
+ for split, label in (("train", "训练集(拟合)"), ("validation", "独立验证集(未参与优化)")):
94
+ lines += [f"## {label}", "", "| 视频序号 | 类型 | 背景语言 | 背景音量 | 控制准确率 | 背景误动率 |",
95
+ "|---|---|---|---|---:|---:|"]
96
+ for row in plan["samples"]:
97
+ if row["split"] != split:
98
+ continue
99
+ number = row["number"]
100
+ link = f"[{number}]({split}/{number}.mp4)" if (split, number) in done else f"{number} 待生成"
101
+ metrics = row["metrics"]; bg = metrics.get("background_only_false_motion_rate")
102
+ bg_text = "无评分区间" if bg is None else f"{bg:.1%}"
103
+ lines.append(f"| {link} | {LABELS[row['profile']]} | {'中文' if row['background_language']=='zh' else '英文'} | "
104
+ f"{'较高' if row['background_high'] else '普通'} | {metrics['control_accuracy']:.1%} | {bg_text} |")
105
+ lines += ["", f"[本组原始逐条指标](../evaluations/final/{split}/report.json)", ""]
106
+ lines += ["单条视频指标用于定位问题,不代替全部验证结果,也不代表机器人实际执行成功。", "",
107
+ "[渲染状态](status.json) · [完整抽样溯源](plan.json)", ""]
108
+ (output / "README.md").write_text("\n".join(lines))
109
+
110
+
111
+ def main():
112
+ parser = argparse.ArgumentParser(description=__doc__)
113
+ parser.add_argument("--run-dir", type=Path, required=True)
114
+ parser.add_argument("--manifest", type=Path, required=True)
115
+ args = parser.parse_args()
116
+ os.environ.update(CUDA_VISIBLE_DEVICES="", OMP_NUM_THREADS="1", MKL_NUM_THREADS="1", PYTHONDONTWRITEBYTECODE="1")
117
+ from scripts.infer.third_party_checkpoint_probe import check_renderer
118
+ plan = prepare(args.run_dir, args.manifest.resolve())
119
+ output = Path(plan["run_dir"]) / "prediction_videos"
120
+ output.mkdir(exist_ok=False)
121
+ write(output / "plan.json", plan)
122
+ for split in ("train", "validation"):
123
+ (output / split).mkdir()
124
+ completed, failures = [], []
125
+ def status(phase):
126
+ write(output / "status.json", {"phase": phase, "time": time.time(), "pid": os.getpid(),
127
+ "completed": len(completed), "total": plan["count"], "failures": failures,
128
+ "gpu_used": False, "inference_repeated": False})
129
+ publish(output, plan, completed)
130
+ status("starting")
131
+ try:
132
+ check_renderer()
133
+ with ThreadPoolExecutor(max_workers=12) as pool:
134
+ jobs = {pool.submit(render_one, row, output / row["split"]): row for row in plan["samples"]}
135
+ for future in as_completed(jobs):
136
+ row = jobs[future]
137
+ try:
138
+ completed.append(future.result())
139
+ print(f"{row['split']}/{row['number']} completed", flush=True)
140
+ except Exception as error:
141
+ failures.append({"split": row["split"], "number": row["number"], "error": repr(error)})
142
+ status("rendering")
143
+ if failures:
144
+ raise RuntimeError("Some final videos failed; inspect status.json and per-sample render.log")
145
+ status("completed")
146
+ except Exception as error:
147
+ failures.append({"error": repr(error)}); status("failed"); raise
148
+
149
+
150
+ if __name__ == "__main__":
151
+ main()
scripts/infer/run_multiturn_infer_suite.py ADDED
@@ -0,0 +1,700 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/usr/bin/env python3
2
+ """Run HFD multi-turn evaluation cases through a dynamic GPU queue."""
3
+
4
+ from __future__ import annotations
5
+
6
+ import hashlib
7
+ import json
8
+ import os
9
+ import re
10
+ import subprocess
11
+ import sys
12
+ import time
13
+ from collections import deque
14
+ from dataclasses import dataclass
15
+ from pathlib import Path
16
+ from typing import Any
17
+
18
+
19
+ PROJECT_ROOT = Path(__file__).resolve().parents[2]
20
+ DEFAULT_SUITE_V23 = PROJECT_ROOT / "scripts/infer/multiturn_auto_suite_v2.json"
21
+ DEFAULT_SUITE_V24 = PROJECT_ROOT / "scripts/infer/multiturn_auto_suite_v3.json"
22
+
23
+
24
+ @dataclass(frozen=True)
25
+ class InferenceCase:
26
+ name: str
27
+ interaction_mode: str
28
+ data_split: str
29
+ scenario_path: Path
30
+ scenario: dict[str, Any]
31
+ interaction_subtype: str = ""
32
+
33
+
34
+ def _truthy(value: str | None) -> bool:
35
+ return str(value or "").strip().lower() in {"1", "true", "yes", "on"}
36
+
37
+
38
+ def _motion_source_id(namespace: str, filename: str) -> str:
39
+ leaf = filename.replace("\\", "/").rstrip("/").rsplit("/", 1)[-1]
40
+ if leaf.lower().endswith(".csv"):
41
+ leaf = leaf[:-4]
42
+ return f"{namespace.strip().strip('/')}/{leaf}"
43
+
44
+
45
+ def _source_is_val(source_id: str, *, split_seed: int, val_ratio: float) -> bool:
46
+ digest = hashlib.sha1(
47
+ f"psi0-motion-source-split-v1\0{split_seed}\0{source_id}".encode()
48
+ ).digest()
49
+ threshold = int(float(val_ratio) * (1 << 64))
50
+ return int.from_bytes(digest[:8], "big", signed=False) < threshold
51
+
52
+
53
+ def _load_json(path: Path) -> dict[str, Any]:
54
+ payload = json.loads(path.read_text(encoding="utf-8"))
55
+ if not isinstance(payload, dict):
56
+ raise ValueError(f"JSON root must be an object: {path}")
57
+ return payload
58
+
59
+
60
+ def validate_scenario(
61
+ path: Path,
62
+ *,
63
+ run_config: dict[str, Any],
64
+ expected_mode: str | None = None,
65
+ expected_split: str | None = None,
66
+ expected_subtype: str | None = None,
67
+ ) -> InferenceCase:
68
+ payload = _load_json(path)
69
+ if payload.get("format") != "hfd_multiturn_auto_infer_scenario_v1":
70
+ raise ValueError(f"Unsupported multi-turn scenario: {path}")
71
+ turns = list(payload.get("turns", []))
72
+ if not 2 <= len(turns) <= 4:
73
+ raise ValueError(f"Scenario must contain 2-4 turns, got {len(turns)}")
74
+ mode = str(payload.get("interaction_mode", "turn-taking")).strip().lower()
75
+ supported_modes = {
76
+ "turn-taking",
77
+ "interruption",
78
+ "emergency-stop",
79
+ "safe-hold",
80
+ "backchannel",
81
+ "hesitation",
82
+ "pause",
83
+ }
84
+ if mode not in supported_modes:
85
+ raise ValueError(f"Unsupported interaction_mode={mode!r}: {path}")
86
+ split = str(payload.get("data_split", "custom")).strip().lower()
87
+ if split not in {"train", "val", "custom"}:
88
+ raise ValueError(f"Unsupported data_split={split!r}: {path}")
89
+ if expected_mode is not None and mode != expected_mode:
90
+ raise ValueError(f"Suite/scenario mode mismatch: {expected_mode} != {mode}")
91
+ if expected_split is not None and split != expected_split:
92
+ raise ValueError(f"Suite/scenario split mismatch: {expected_split} != {split}")
93
+ subtype = str(payload.get("interaction_subtype", "")).strip().lower()
94
+ valid_subtypes = {
95
+ "hesitation": {"filled-pause", "self-repair"},
96
+ "pause": {"silent-pause"},
97
+ }
98
+ if mode in valid_subtypes:
99
+ if subtype not in valid_subtypes[mode]:
100
+ raise ValueError(
101
+ f"Invalid {mode} interaction_subtype={subtype!r}: {path}"
102
+ )
103
+ elif subtype:
104
+ raise ValueError(
105
+ f"interaction_subtype is only valid for hesitation/pause: {path}"
106
+ )
107
+ if expected_subtype is not None and subtype != expected_subtype:
108
+ raise ValueError(
109
+ f"Suite/scenario subtype mismatch: {expected_subtype} != {subtype}"
110
+ )
111
+
112
+ gaps = [int(value) for value in payload["inter_turn_background_steps_by_turn"]]
113
+ if len(gaps) != len(turns) - 1 or any(value < 0 for value in gaps):
114
+ raise ValueError(f"Invalid per-turn background gaps: {gaps}")
115
+ declared_gap_seconds = [
116
+ float(value)
117
+ for value in payload.get("inter_turn_background_seconds_by_turn", [])
118
+ ]
119
+ if declared_gap_seconds:
120
+ expected_gap_seconds = [0.4 * value for value in gaps]
121
+ if len(declared_gap_seconds) != len(gaps) or any(
122
+ abs(observed - expected) > 1e-9
123
+ for observed, expected in zip(
124
+ declared_gap_seconds,
125
+ expected_gap_seconds,
126
+ )
127
+ ):
128
+ raise ValueError(
129
+ "Scenario gap seconds disagree with 0.4-second Qwen steps: "
130
+ f"{declared_gap_seconds} != {expected_gap_seconds}"
131
+ )
132
+ if mode in {"interruption", "emergency-stop", "safe-hold"}:
133
+ gap_seconds = [0.4 * value for value in gaps]
134
+ if any(value < 2.0 or value > 6.0 for value in gap_seconds):
135
+ raise ValueError(
136
+ "Mid-motion command gaps must stay within 2-6 seconds: "
137
+ f"{gap_seconds}"
138
+ )
139
+ if mode == "backchannel":
140
+ marked = [
141
+ index
142
+ for index, turn in enumerate(turns)
143
+ if str(turn.get("backchannel_intent", "")).strip()
144
+ ]
145
+ if len(marked) != 1 or marked[0] == 0:
146
+ raise ValueError(
147
+ "Backchannel scenario needs exactly one acknowledgement after "
148
+ f"a real action command: indices={marked}"
149
+ )
150
+ if str(turns[marked[0]]["backchannel_intent"]).strip() not in {
151
+ "continuer",
152
+ "assessment",
153
+ }:
154
+ raise ValueError("Unknown backchannel intent category")
155
+ voice_stems = {
156
+ Path(str(turn["audio_path"])).stem for turn in turns
157
+ }
158
+ if len(voice_stems) != 1:
159
+ raise ValueError(
160
+ "Backchannel scenario must keep one speaker/language identity: "
161
+ f"{sorted(voice_stems)}"
162
+ )
163
+ onset_gap_seconds = 0.4 * gaps[marked[0] - 1]
164
+ if not 1.2 <= onset_gap_seconds <= 6.0:
165
+ raise ValueError(
166
+ "Backchannel must arrive during active motion: "
167
+ f"gap={onset_gap_seconds}s"
168
+ )
169
+
170
+ window_counts = [
171
+ int(value) for value in payload.get("turn_audio_window_counts", [])
172
+ ]
173
+ expected_starts = [
174
+ int(value) for value in payload.get("expected_turn_window_starts", [])
175
+ ]
176
+ if window_counts or expected_starts:
177
+ if len(window_counts) != len(turns) or len(expected_starts) != len(turns):
178
+ raise ValueError("Timeline audit needs one window count/start per turn")
179
+ derived_starts = [0]
180
+ for count, gap in zip(window_counts, gaps):
181
+ if count <= 0:
182
+ raise ValueError(f"Invalid audio window counts: {window_counts}")
183
+ derived_starts.append(derived_starts[-1] + count + gap)
184
+ if derived_starts != expected_starts:
185
+ raise ValueError(
186
+ f"Scenario gap timeline mismatch: {derived_starts} != {expected_starts}"
187
+ )
188
+
189
+ source_starts = [
190
+ int(value)
191
+ for value in payload.get("source_turn_speech_start_frames_50fps", [])
192
+ ]
193
+ if source_starts:
194
+ if len(source_starts) != len(turns) or not expected_starts:
195
+ raise ValueError(
196
+ "Source timeline audit needs one source start and one expected "
197
+ "inference start per turn"
198
+ )
199
+ relative_frames = [value - source_starts[0] for value in source_starts]
200
+ if any(value % 20 != 0 for value in relative_frames):
201
+ raise ValueError(
202
+ "Source speech starts are not aligned to the 20-frame Qwen grid: "
203
+ f"{source_starts}"
204
+ )
205
+ source_relative_steps = [value // 20 for value in relative_frames]
206
+ if source_relative_steps != expected_starts:
207
+ raise ValueError(
208
+ "Scenario inference starts do not reproduce the source speech "
209
+ f"timeline: {source_relative_steps} != {expected_starts}"
210
+ )
211
+
212
+ data = dict(run_config.get("data", {}))
213
+ train = dict(run_config.get("train", {}))
214
+ active_semantics = int(train.get("full_duplex_semantics_version", 0))
215
+ required_semantics = int(payload.get("required_semantics_version", 0))
216
+ if active_semantics < required_semantics:
217
+ raise ValueError(
218
+ f"Scenario requires semantics {required_semantics}, run has {active_semantics}"
219
+ )
220
+ if mode == "interruption" and (
221
+ active_semantics < 20 or not bool(data.get("interruption_sessions", False))
222
+ ):
223
+ raise ValueError("Interruption inference requires a semantics-v20 run")
224
+ if mode in {"emergency-stop", "safe-hold"}:
225
+ variants = {
226
+ "emergency-stop": int(
227
+ data.get("emergency_stop_session_variants", 0)
228
+ ),
229
+ "safe-hold": int(data.get("safe_hold_session_variants", 0)),
230
+ }
231
+ if (
232
+ active_semantics < 21
233
+ or not bool(data.get("independent_task_pools", False))
234
+ or variants[mode] <= 0
235
+ ):
236
+ raise ValueError(
237
+ f"{mode} inference requires an enabled semantics-v21 task pool"
238
+ )
239
+ control_turns = [
240
+ turn
241
+ for turn in turns
242
+ if str(turn.get("control_intent", "")).strip()
243
+ ]
244
+ if len(control_turns) != 1 or str(
245
+ control_turns[0].get("control_intent")
246
+ ).strip() != mode:
247
+ raise ValueError(
248
+ f"{mode} scenario needs exactly one matching control command"
249
+ )
250
+ if mode == "backchannel":
251
+ if (
252
+ active_semantics < 22
253
+ or not bool(data.get("independent_task_pools", False))
254
+ or int(data.get("backchannel_session_variants", 0)) <= 0
255
+ ):
256
+ raise ValueError(
257
+ "Backchannel inference requires an enabled semantics-v22 pool"
258
+ )
259
+ if mode in {"hesitation", "pause"}:
260
+ variants = {
261
+ "hesitation": int(data.get("hesitation_session_variants", 0)),
262
+ "pause": int(data.get("pause_session_variants", 0)),
263
+ }
264
+ if (
265
+ active_semantics < 23
266
+ or not bool(data.get("independent_task_pools", False))
267
+ or variants[mode] <= 0
268
+ ):
269
+ raise ValueError(
270
+ f"{mode} inference requires an enabled semantics-v23 task pool"
271
+ )
272
+ event = dict(payload.get("disfluency_event", {}))
273
+ if str(event.get("subtype", "")).strip().lower() != subtype:
274
+ raise ValueError(
275
+ f"{mode} scenario has no matching disfluency_event: {path}"
276
+ )
277
+ gap_steps = int(event.get("gap_steps", 0))
278
+ if not 1 <= gap_steps <= 7:
279
+ raise ValueError(
280
+ f"{mode} gap must use 1-7 Qwen steps, got {gap_steps}"
281
+ )
282
+ event_turn = int(event.get("turn_index", -1))
283
+ if not 0 <= event_turn < len(turns):
284
+ raise ValueError(f"Invalid disfluency event turn: {event_turn}")
285
+ if str(turns[event_turn].get("disfluency_subtype", "")) != subtype:
286
+ raise ValueError(
287
+ "Scenario event turn does not carry the declared disfluency subtype"
288
+ )
289
+ if active_semantics >= 24 and subtype == "self-repair":
290
+ repair_style = str(event.get("repair_style", "")).strip().lower()
291
+ if repair_style not in {"explicit-cue", "implicit-restart"}:
292
+ raise ValueError(
293
+ "Semantics-v24 self-repair scenario must declare "
294
+ f"repair_style, got {repair_style!r}"
295
+ )
296
+ marker_id = str(event.get("marker_id", "")).strip()
297
+ if (repair_style == "explicit-cue") != bool(marker_id):
298
+ raise ValueError(
299
+ "Explicit self-repair requires a marker and implicit "
300
+ "restart must not carry one"
301
+ )
302
+
303
+ if split != "custom":
304
+ split_seed = int(data.get("split_seed", -1))
305
+ val_ratio = float(data.get("val_ratio", -1.0))
306
+ namespace = str(data.get("split_source_namespace", "")).strip().strip("/")
307
+ expected_namespace = str(
308
+ payload.get("split_source_namespace", "")
309
+ ).strip().strip("/")
310
+ if split_seed != int(payload.get("split_seed", -2)):
311
+ raise ValueError("Scenario/run split_seed mismatch")
312
+ if abs(val_ratio - float(payload.get("val_ratio", -2.0))) > 1e-12:
313
+ raise ValueError("Scenario/run val_ratio mismatch")
314
+ if not namespace or namespace != expected_namespace:
315
+ raise ValueError("Scenario/run split namespace mismatch")
316
+ for turn in turns:
317
+ filename = str(turn.get("source_motion_filename", "")).strip()
318
+ # Stop/hold utterances are control events, not new primitive
319
+ # motions. Their TTS path is audited above and deliberately has
320
+ # no train/val motion identity of its own.
321
+ if not filename and (
322
+ str(turn.get("control_intent", "")).strip()
323
+ or str(turn.get("backchannel_intent", "")).strip()
324
+ ):
325
+ continue
326
+ if not filename:
327
+ raise ValueError("Split-tagged scenario turn has no motion filename")
328
+ observed = (
329
+ "val"
330
+ if _source_is_val(
331
+ _motion_source_id(namespace, filename),
332
+ split_seed=split_seed,
333
+ val_ratio=val_ratio,
334
+ )
335
+ else "train"
336
+ )
337
+ if observed != split:
338
+ raise ValueError(
339
+ f"Scenario source split mismatch: {filename} is {observed}, "
340
+ f"scenario claims {split}"
341
+ )
342
+
343
+ for turn in turns:
344
+ audio_path = Path(str(turn["audio_path"])).expanduser().resolve()
345
+ if not audio_path.is_file():
346
+ raise FileNotFoundError(f"Missing scenario audio: {audio_path}")
347
+ return InferenceCase(
348
+ name=str(payload["name"]),
349
+ interaction_mode=mode,
350
+ data_split=split,
351
+ scenario_path=path.resolve(),
352
+ scenario=payload,
353
+ interaction_subtype=subtype,
354
+ )
355
+
356
+
357
+ def load_cases(
358
+ suite_path: Path,
359
+ *,
360
+ run_config: dict[str, Any],
361
+ single_scenario: Path | None = None,
362
+ ) -> list[InferenceCase]:
363
+ if single_scenario is not None:
364
+ return [validate_scenario(single_scenario, run_config=run_config)]
365
+ suite = _load_json(suite_path)
366
+ if suite.get("format") not in {
367
+ "hfd_multiturn_auto_infer_suite_v2",
368
+ "hfd_multiturn_auto_infer_suite_v3",
369
+ }:
370
+ raise ValueError(f"Unsupported inference suite: {suite_path}")
371
+ data = dict(run_config.get("data", {}))
372
+ enabled_variants = {
373
+ "turn-taking": int(data.get("turn_taking_session_variants", 0)),
374
+ "interruption": int(
375
+ data.get("command_interruption_session_variants", 0)
376
+ ),
377
+ "emergency-stop": int(
378
+ data.get("emergency_stop_session_variants", 0)
379
+ ),
380
+ "safe-hold": int(data.get("safe_hold_session_variants", 0)),
381
+ "backchannel": int(data.get("backchannel_session_variants", 0)),
382
+ "hesitation": int(data.get("hesitation_session_variants", 0)),
383
+ "pause": int(data.get("pause_session_variants", 0)),
384
+ }
385
+ enabled_modes = {
386
+ mode for mode, variants in enabled_variants.items() if variants > 0
387
+ }
388
+ cases: list[InferenceCase] = []
389
+ for item in suite.get("cases", []):
390
+ item_mode = str(item["interaction_mode"])
391
+ if item_mode not in enabled_modes:
392
+ continue
393
+ scenario_path = Path(str(item["scenario_file"]))
394
+ if not scenario_path.is_absolute():
395
+ scenario_path = (suite_path.parent / scenario_path).resolve()
396
+ cases.append(
397
+ validate_scenario(
398
+ scenario_path,
399
+ run_config=run_config,
400
+ expected_mode=item_mode,
401
+ expected_split=str(item["data_split"]),
402
+ expected_subtype=str(item.get("interaction_subtype", "")),
403
+ )
404
+ )
405
+ identities = {
406
+ (case.interaction_mode, case.interaction_subtype, case.data_split)
407
+ for case in cases
408
+ }
409
+ expected = set()
410
+ for mode in enabled_modes:
411
+ subtypes = (
412
+ ("filled-pause", "self-repair")
413
+ if mode == "hesitation"
414
+ else ("silent-pause",)
415
+ if mode == "pause"
416
+ else ("",)
417
+ )
418
+ expected.update(
419
+ (mode, subtype, split)
420
+ for subtype in subtypes
421
+ for split in ("train", "val")
422
+ )
423
+ if identities != expected:
424
+ raise ValueError(
425
+ "Suite must contain train/val cases for every enabled task: "
426
+ f"expected={expected} observed={identities}"
427
+ )
428
+ return cases
429
+
430
+
431
+ def checkpoint_ready(path: Path) -> bool:
432
+ required = (
433
+ path / "model.safetensors",
434
+ path / "hfd_fused_delta_manifest.json",
435
+ path / "full_duplex_source_boundary.json",
436
+ )
437
+ if any(not item.is_file() or item.stat().st_size <= 0 for item in required):
438
+ return False
439
+ try:
440
+ marker = _load_json(required[-1])
441
+ except Exception:
442
+ return False
443
+ return bool(marker.get("source_group_complete", False))
444
+
445
+
446
+ def ready_checkpoint_steps(run_dir: Path) -> list[int]:
447
+ checkpoints = run_dir / "checkpoints"
448
+ found: list[int] = []
449
+ for path in checkpoints.glob("ckpt_[0-9]*"):
450
+ match = re.fullmatch(r"ckpt_([0-9]+)", path.name)
451
+ if match and path.is_dir() and checkpoint_ready(path):
452
+ found.append(int(match.group(1)))
453
+ return sorted(found)
454
+
455
+
456
+ def visible_gpus() -> list[str]:
457
+ configured = os.environ.get("HFD_INFER_GPUS", "").strip()
458
+ if not configured:
459
+ configured = os.environ.get("CUDA_VISIBLE_DEVICES", "").strip()
460
+ if configured:
461
+ values = [value.strip() for value in configured.split(",") if value.strip()]
462
+ else:
463
+ result = subprocess.run(
464
+ ["nvidia-smi", "--query-gpu=index", "--format=csv,noheader"],
465
+ check=False,
466
+ text=True,
467
+ capture_output=True,
468
+ )
469
+ values = [line.strip() for line in result.stdout.splitlines() if line.strip()]
470
+ if not values:
471
+ raise RuntimeError("No inference GPU is visible")
472
+ if len(values) != len(set(values)):
473
+ raise ValueError(f"Duplicate GPU identifiers: {values}")
474
+ return values
475
+
476
+
477
+ def _safe_component(value: str) -> str:
478
+ return re.sub(r"[^A-Za-z0-9_.-]+", "_", value)
479
+
480
+
481
+ def case_result_dir(run_dir: Path, case: InferenceCase, step: int) -> Path:
482
+ turns = list(case.scenario["turns"])
483
+ # ``infer_render_auto.sh`` historically sanitizes ``echo "$stem"`` with
484
+ # ``tr -cs``. The terminating newline therefore becomes one trailing
485
+ # underscore. Reproduce that established on-disk name here rather than
486
+ # silently renaming or re-rendering every existing inference result.
487
+ audio_stem = _safe_component(Path(str(turns[0]["audio_path"])).stem + "\n")
488
+ sample_name = (
489
+ f"00_{_safe_component(case.name)}_{_safe_component(audio_stem)}_"
490
+ f"session{len(turns)}turn"
491
+ )
492
+ output_group = Path(case.interaction_mode)
493
+ if case.interaction_mode == "hesitation":
494
+ output_group /= (
495
+ "filled" if case.interaction_subtype == "filled-pause" else "self-repair"
496
+ )
497
+ return (
498
+ run_dir
499
+ / output_group
500
+ / f"ckpt_{step}"
501
+ / case.data_split
502
+ / sample_name
503
+ )
504
+
505
+
506
+ def case_settled(run_dir: Path, case: InferenceCase, step: int) -> bool:
507
+ result_dir = case_result_dir(run_dir, case, step)
508
+ return (result_dir / ".complete").is_file() or (result_dir / ".failed").is_file()
509
+
510
+
511
+ def case_failed(run_dir: Path, case: InferenceCase, step: int) -> bool:
512
+ return (case_result_dir(run_dir, case, step) / ".failed").is_file()
513
+
514
+
515
+ def case_environment(
516
+ run_dir: Path,
517
+ case: InferenceCase,
518
+ *,
519
+ step: int,
520
+ gpu: str,
521
+ ) -> dict[str, str]:
522
+ payload = case.scenario
523
+ turns = list(payload["turns"])
524
+ audio_paths = [str(Path(str(turn["audio_path"])).expanduser().resolve()) for turn in turns]
525
+ environment = os.environ.copy()
526
+ output_group = Path(case.interaction_mode)
527
+ if case.interaction_mode == "hesitation":
528
+ output_group /= (
529
+ "filled" if case.interaction_subtype == "filled-pause" else "self-repair"
530
+ )
531
+ environment.update(
532
+ {
533
+ "CUDA_VISIBLE_DEVICES": gpu,
534
+ "HFD_INFER_GPUS": gpu,
535
+ "HFD_INFER_DEVICE": "cuda:0",
536
+ "HFD_INFER_ONCE": "1",
537
+ "HFD_INFER_EXACT_CHECKPOINT_STEP": str(step),
538
+ "HFD_INFER_MIN_CHECKPOINT_STEP": str(step),
539
+ "HFD_INFER_OUTPUT_ROOT": str(run_dir / output_group),
540
+ "HFD_INFER_SAMPLE_PARENT": case.data_split,
541
+ "HFD_INFER_SAMPLE_TAG": case.name,
542
+ "HFD_INFER_SCENARIO_FILE": str(case.scenario_path),
543
+ "HFD_INFER_AUDIO_PATHS": audio_paths[0],
544
+ "HFD_INFER_ADDITIONAL_AUDIO_PATHS": ",".join(audio_paths[1:]),
545
+ "HFD_INFER_INITIAL_IDLE_STEPS": str(payload["initial_idle_steps"]),
546
+ "HFD_INFER_MOTION_FRAMES": str(payload["motion_frames_per_turn"]),
547
+ "HFD_INFER_INTER_TURN_BACKGROUND_STEPS_BY_TURN": ",".join(
548
+ str(int(value))
549
+ for value in payload["inter_turn_background_steps_by_turn"]
550
+ ),
551
+ "HFD_INFER_TERMINAL_BACKGROUND_STEPS": str(
552
+ payload["terminal_background_steps"]
553
+ ),
554
+ }
555
+ )
556
+ init_motion = str(payload.get("init_motion_dir", "")).strip()
557
+ if init_motion:
558
+ environment["HFD_INFER_INIT_MOTION_DIRS"] = init_motion
559
+ return environment
560
+
561
+
562
+ def run_dynamic_queue(
563
+ run_dir: Path,
564
+ cases: list[InferenceCase],
565
+ *,
566
+ step: int,
567
+ gpus: list[str],
568
+ ) -> None:
569
+ pending = deque(case for case in cases if not case_settled(run_dir, case, step))
570
+ available = deque(gpus)
571
+ running: dict[subprocess.Popen[bytes], tuple[InferenceCase, str]] = {}
572
+ failures: list[str] = []
573
+ base_script = PROJECT_ROOT / "infer_render_auto.sh"
574
+ while pending or running:
575
+ while pending and available:
576
+ case = pending.popleft()
577
+ gpu = available.popleft()
578
+ print(
579
+ f"[suite] start ckpt_{step} {case.interaction_mode}/{case.data_split} "
580
+ f"on GPU {gpu}",
581
+ flush=True,
582
+ )
583
+ process = subprocess.Popen(
584
+ ["bash", str(base_script), str(run_dir)],
585
+ cwd=PROJECT_ROOT,
586
+ env=case_environment(run_dir, case, step=step, gpu=gpu),
587
+ )
588
+ running[process] = (case, gpu)
589
+ finished = [process for process in running if process.poll() is not None]
590
+ if not finished:
591
+ time.sleep(0.25)
592
+ continue
593
+ for process in finished:
594
+ case, gpu = running.pop(process)
595
+ available.append(gpu)
596
+ if process.returncode != 0:
597
+ failures.append(
598
+ f"{case.interaction_mode}/{case.data_split} "
599
+ f"exited {process.returncode}"
600
+ )
601
+ elif case_failed(run_dir, case, step):
602
+ failures.append(
603
+ f"{case.interaction_mode}/{case.data_split} wrote .failed"
604
+ )
605
+ elif not case_settled(run_dir, case, step):
606
+ failures.append(
607
+ f"{case.interaction_mode}/{case.data_split} wrote no marker"
608
+ )
609
+ else:
610
+ print(
611
+ f"[suite] done ckpt_{step} "
612
+ f"{case.interaction_mode}/{case.data_split} on GPU {gpu}",
613
+ flush=True,
614
+ )
615
+ if failures:
616
+ raise RuntimeError(
617
+ f"ckpt_{step} inference suite failed: " + "; ".join(failures)
618
+ )
619
+
620
+
621
+ def main() -> None:
622
+ if len(sys.argv) != 2:
623
+ raise SystemExit(f"usage: {Path(sys.argv[0]).name} RUN_DIR")
624
+ run_dir = Path(sys.argv[1]).expanduser().resolve()
625
+ run_config_path = run_dir / "run_config.json"
626
+ if not run_config_path.is_file():
627
+ raise SystemExit(f"Invalid HFD run directory: {run_dir}")
628
+ run_config = _load_json(run_config_path)
629
+ configured_suite = os.environ.get("HFD_MULTITURN_SUITE_FILE", "").strip()
630
+ active_semantics = int(
631
+ dict(run_config.get("train", {})).get(
632
+ "full_duplex_semantics_version",
633
+ 0,
634
+ )
635
+ )
636
+ suite_path = Path(
637
+ configured_suite
638
+ or (DEFAULT_SUITE_V24 if active_semantics >= 24 else DEFAULT_SUITE_V23)
639
+ )
640
+ suite_path = suite_path.expanduser().resolve()
641
+ single_raw = os.environ.get("HFD_MULTITURN_SCENARIO_FILE", "").strip()
642
+ single_scenario = Path(single_raw).expanduser().resolve() if single_raw else None
643
+ cases = load_cases(
644
+ suite_path,
645
+ run_config=run_config,
646
+ single_scenario=single_scenario,
647
+ )
648
+ gpus = visible_gpus()
649
+ check_interval = max(int(os.environ.get("HFD_INFER_CHECK_INTERVAL", "30")), 1)
650
+ infer_once = _truthy(os.environ.get("HFD_INFER_ONCE"))
651
+ exact_raw = os.environ.get("HFD_INFER_EXACT_CHECKPOINT_STEP", "").strip()
652
+ exact_step = int(exact_raw) if exact_raw else None
653
+ minimum_raw = os.environ.get("HFD_INFER_MIN_CHECKPOINT_STEP", "auto").strip()
654
+ initial_steps = ready_checkpoint_steps(run_dir)
655
+ if minimum_raw == "auto":
656
+ minimum_step = initial_steps[-1] if initial_steps else 0
657
+ else:
658
+ minimum_step = int(minimum_raw)
659
+ print(
660
+ f"AUTO INFER SUITE: cases={len(cases)} GPUs={','.join(gpus)} "
661
+ f"min_step={minimum_step} exact_step={exact_step or 'none'}",
662
+ flush=True,
663
+ )
664
+
665
+ while True:
666
+ ready = ready_checkpoint_steps(run_dir)
667
+ if exact_step is not None:
668
+ candidates = [exact_step] if exact_step in ready else []
669
+ else:
670
+ candidates = [step for step in ready if step >= minimum_step]
671
+ pending_steps = [
672
+ step
673
+ for step in candidates
674
+ if any(not case_settled(run_dir, case, step) for case in cases)
675
+ ]
676
+ if pending_steps:
677
+ step = max(pending_steps)
678
+ run_dynamic_queue(run_dir, cases, step=step, gpus=gpus)
679
+ if infer_once or exact_step is not None:
680
+ break
681
+ else:
682
+ failed = [
683
+ f"{case.interaction_mode}/{case.data_split}"
684
+ for step in candidates
685
+ for case in cases
686
+ if case_failed(run_dir, case, step)
687
+ ]
688
+ if failed:
689
+ raise RuntimeError(
690
+ "Existing inference failure marker(s) require inspection or "
691
+ f"cleanup before retry: {', '.join(failed)}"
692
+ )
693
+ print(f"[{time.strftime('%F %T')}] no pending checkpoint", flush=True)
694
+ if infer_once or exact_step is not None:
695
+ break
696
+ time.sleep(check_interval)
697
+
698
+
699
+ if __name__ == "__main__":
700
+ main()
scripts/infer/seed_audio_to_motion.py ADDED
@@ -0,0 +1,530 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/usr/bin/env python3
2
+ from __future__ import annotations
3
+
4
+ import argparse
5
+ import json
6
+ import math
7
+ import random
8
+ import re
9
+ import shutil
10
+ import subprocess
11
+ from pathlib import Path
12
+ from typing import List
13
+
14
+ import numpy as np
15
+ import torch
16
+
17
+ from psi.config.config import LaunchConfig
18
+ from psi.data.seed.motion_repr import quat_inv_wxyz, quat_mul_wxyz, recover_seed_abs_from_textop_action
19
+ from psi.data.seed.progress import build_progress_conditions
20
+ from psi.models.psi0 import Psi0Model
21
+ from psi.utils import parse_args_to_tyro_config, seed_everything
22
+
23
+ from seed_text_to_motion import (
24
+ TARGET_FPS,
25
+ build_model_states_from_abs,
26
+ finite_difference,
27
+ load_pose_from_motion_dir,
28
+ load_source_frame_count,
29
+ normalize_quaternion_rows,
30
+ quaternion_angular_velocity,
31
+ recover_seed_abs_from_textop_action_with_joint_delta,
32
+ resolve_ckpt_step,
33
+ sanitize_motion_name,
34
+ should_decode_textop_joint_from_delta,
35
+ write_csv,
36
+ write_metadata,
37
+ )
38
+
39
+
40
+ def cfg_get(obj, key: str, default=None):
41
+ if isinstance(obj, dict):
42
+ return obj.get(key, default)
43
+ return getattr(obj, key, default)
44
+
45
+
46
+ def audio_data_cfg(launch_config: LaunchConfig, key: str = "audio"):
47
+ return cfg_get(launch_config.data, key, launch_config.data)
48
+
49
+
50
+ def load_audio_array(video_path: Path, sampling_rate: int) -> np.ndarray:
51
+ cmd = [
52
+ "ffmpeg",
53
+ "-v",
54
+ "error",
55
+ "-i",
56
+ str(video_path),
57
+ "-ac",
58
+ "1",
59
+ "-ar",
60
+ str(sampling_rate),
61
+ "-f",
62
+ "f32le",
63
+ "-",
64
+ ]
65
+ proc = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, check=False)
66
+ if proc.returncode != 0:
67
+ msg = proc.stderr.decode("utf-8", errors="ignore")
68
+ raise RuntimeError(f"ffmpeg failed to decode audio from {video_path}: {msg}")
69
+ audio = np.frombuffer(proc.stdout, dtype=np.float32).copy()
70
+ if audio.size == 0:
71
+ raise RuntimeError(f"No audio samples decoded from {video_path}")
72
+ return audio
73
+
74
+
75
+ def audio_segment(audio: np.ndarray, sampling_rate: int, start_sec: float, duration_sec: float) -> np.ndarray:
76
+ num_samples = max(1, int(round(duration_sec * sampling_rate)))
77
+ start_sample = int(round(start_sec * sampling_rate))
78
+ end_sample = start_sample + num_samples
79
+
80
+ segment = np.zeros((num_samples,), dtype=np.float32)
81
+ src_start = max(0, start_sample)
82
+ src_end = min(audio.shape[0], end_sample)
83
+ if src_end > src_start:
84
+ dst_start = src_start - start_sample
85
+ segment[dst_start : dst_start + (src_end - src_start)] = audio[src_start:src_end]
86
+ return segment
87
+
88
+
89
+ def read_start_hold_seconds(motion_dir: Path, align_start_hold: bool) -> float:
90
+ if not align_start_hold:
91
+ return 0.0
92
+ info_path = motion_dir / "info.txt"
93
+ if not info_path.exists():
94
+ return 0.0
95
+ text = info_path.read_text(encoding="utf-8", errors="ignore")
96
+ match = re.search(r"start_hold_seconds:\s*([0-9.]+)", text)
97
+ return float(match.group(1)) if match else 0.0
98
+
99
+
100
+ def resolve_motion_dir(root_dir: Path, motion_dir_arg: str) -> Path:
101
+ motion_dir = Path(motion_dir_arg).expanduser()
102
+ if motion_dir.is_absolute():
103
+ return motion_dir
104
+ return root_dir / motion_dir
105
+
106
+
107
+ def resolve_video_path(video_dir: Path, motion_dir: Path, video_path_arg: str) -> Path:
108
+ if video_path_arg:
109
+ video_path = Path(video_path_arg).expanduser()
110
+ if video_path.is_absolute():
111
+ return video_path
112
+ return video_dir / video_path
113
+ return video_dir / f"{motion_dir.name}.mp4"
114
+
115
+
116
+ def resolve_audio_path(audio_path: Path, extensions: str = ".wav,.flac,.mp3,.m4a,.aac,.ogg") -> Path:
117
+ if audio_path.is_file():
118
+ return audio_path
119
+ if not audio_path.is_dir():
120
+ return audio_path
121
+ audio_exts = tuple(
122
+ ext if ext.startswith(".") else f".{ext}"
123
+ for ext in [part.strip().lower() for part in str(extensions).split(",")]
124
+ if ext
125
+ )
126
+ candidates = sorted(p for p in audio_path.iterdir() if p.is_file() and p.suffix.lower() in audio_exts)
127
+ if not candidates:
128
+ raise FileNotFoundError(f"No audio files found under {audio_path}")
129
+ return random.choice(candidates)
130
+
131
+
132
+ def normalize_action_for_rtc(maxmin, action: np.ndarray) -> np.ndarray:
133
+ action_min = np.asarray(maxmin.action_min, dtype=np.float32)
134
+ action_max = np.asarray(maxmin.action_max, dtype=np.float32)
135
+ ill_mask = np.abs(action_max - action_min) < 1e-4 * (np.abs(action_max) + np.abs(action_min) + 1e-8)
136
+ action_max = action_max.copy()
137
+ action_max[ill_mask] = 1.0
138
+ action_norm = np.where(ill_mask, action, (action - action_min) / (action_max - action_min) * 2 - 1)
139
+ if bool(getattr(maxmin, "use_norm_mask", False)):
140
+ action_norm_masks = np.asarray(maxmin.action_norm_masks, dtype=bool)
141
+ action_norm = np.where(action_norm_masks, action_norm, action)
142
+ return np.clip(action_norm, -1.0, 1.0).astype(np.float32)
143
+
144
+
145
+ def rollout_audio_motion(
146
+ *,
147
+ model: Psi0Model,
148
+ maxmin,
149
+ odim: int,
150
+ device: str,
151
+ instruction: str,
152
+ audio: np.ndarray,
153
+ audio_sampling_rate: int,
154
+ audio_window_seconds: float,
155
+ audio_left_context_seconds: float,
156
+ full_audio: bool,
157
+ start_hold_seconds: float,
158
+ normalize_state: bool,
159
+ textop_actions: bool,
160
+ decode_textop_joint_from_delta: bool,
161
+ num_past_frames: int,
162
+ use_future_progress_embedding: bool,
163
+ enable_rtc: bool,
164
+ action_exec_horizon: int,
165
+ rtc_max_delay: int,
166
+ num_inference_steps: int,
167
+ target_frames: int,
168
+ init_abs_state: np.ndarray,
169
+ ) -> tuple[np.ndarray, np.ndarray, int]:
170
+ horizon = int(model.action_horizon)
171
+ exec_horizon = int(action_exec_horizon) if enable_rtc else horizon
172
+ if exec_horizon <= 0 or exec_horizon > horizon:
173
+ raise ValueError(f"Invalid action_exec_horizon={exec_horizon}, action_horizon={horizon}")
174
+ rtc_delay = horizon - exec_horizon
175
+ if enable_rtc and rtc_delay > 0 and rtc_delay >= int(rtc_max_delay):
176
+ raise ValueError(
177
+ f"RTC requires action_horizon - action_exec_horizon < max_delay, "
178
+ f"got {horizon} - {exec_horizon} = {rtc_delay}, max_delay={rtc_max_delay}"
179
+ )
180
+ if not textop_actions:
181
+ raise ValueError("This audio inference path is for the 39D TextOp action checkpoint.")
182
+ if target_frames <= 0:
183
+ target_frames = max(exec_horizon, int(round(audio.shape[0] / float(audio_sampling_rate) * TARGET_FPS)))
184
+
185
+ num_chunks = int(math.ceil(float(target_frames) / float(exec_horizon)))
186
+ current_abs = np.asarray(init_abs_state, dtype=np.float32).copy()
187
+ state_horizon = int(num_past_frames) + 1
188
+ state_history: List[np.ndarray] = [current_abs.copy() for _ in range(state_horizon)]
189
+ all_abs: List[np.ndarray] = []
190
+ all_delta: List[np.ndarray] = []
191
+ prev_actions_norm: np.ndarray | None = None
192
+ remaining_frames = int(target_frames)
193
+
194
+ for _ in range(num_chunks):
195
+ frame_cursor = target_frames - remaining_frames
196
+ states = build_model_states_from_abs(
197
+ state_history[-state_horizon:],
198
+ textop_actions=textop_actions,
199
+ odim=odim,
200
+ )
201
+ if normalize_state:
202
+ states = maxmin.normalize_state_func(states)
203
+ states_t = torch.from_numpy(states).unsqueeze(0).to(device)
204
+ future_progress_t = None
205
+ if use_future_progress_embedding:
206
+ _, future_progress = build_progress_conditions(
207
+ frame_id=frame_cursor,
208
+ total_frames=target_frames,
209
+ action_horizon=horizon,
210
+ )
211
+ future_progress_t = torch.from_numpy(future_progress[None]).to(device)
212
+
213
+ # <modify_full_audio> 和训练一致:full_audio=True时每个chunk都看到整段音频embedding,不再给time/progress。
214
+ if full_audio:
215
+ audio_clip = audio.astype(np.float32)
216
+ else:
217
+ audio_start_sec = (
218
+ frame_cursor / float(TARGET_FPS)
219
+ - float(start_hold_seconds)
220
+ - float(audio_left_context_seconds)
221
+ )
222
+ audio_clip = audio_segment(audio, audio_sampling_rate, audio_start_sec, audio_window_seconds)
223
+
224
+ with torch.inference_mode():
225
+ if enable_rtc and prev_actions_norm is not None and rtc_delay > 0:
226
+ prev_rtc = np.concatenate(
227
+ [
228
+ prev_actions_norm[exec_horizon:, :],
229
+ np.zeros((exec_horizon, prev_actions_norm.shape[1]), dtype=np.float32),
230
+ ],
231
+ axis=0,
232
+ )
233
+ prev_actions_t = torch.from_numpy(prev_rtc[None]).to(device)
234
+ pred_norm = model.predict_action_with_training_rtc_flow(
235
+ observations=None,
236
+ states=states_t,
237
+ instructions=[instruction],
238
+ num_inference_steps=num_inference_steps,
239
+ traj2ds=None,
240
+ prev_actions=prev_actions_t,
241
+ inference_delay=rtc_delay,
242
+ max_delay=int(rtc_max_delay),
243
+ audios=[audio_clip],
244
+ future_progress=future_progress_t,
245
+ )[0].float().cpu().numpy()
246
+ else:
247
+ pred_norm = model.predict_action(
248
+ observations=None,
249
+ states=states_t,
250
+ instructions=[instruction],
251
+ num_inference_steps=num_inference_steps,
252
+ traj2ds=None,
253
+ audios=[audio_clip],
254
+ future_progress=future_progress_t,
255
+ )[0].float().cpu().numpy()
256
+
257
+ pred_norm_clipped = np.clip(np.asarray(pred_norm, dtype=np.float32), -1.0, 1.0)
258
+ chunk_pred = np.asarray(maxmin.denormalize(pred_norm_clipped), dtype=np.float32)
259
+
260
+ if decode_textop_joint_from_delta:
261
+ chunk_abs = recover_seed_abs_from_textop_action_with_joint_delta(chunk_pred, current_abs)
262
+ else:
263
+ chunk_abs = recover_seed_abs_from_textop_action(chunk_pred, current_abs)
264
+
265
+ prev_abs = np.concatenate([current_abs[None, :], chunk_abs[:-1]], axis=0)
266
+ joint_delta = chunk_abs[:, :29] - prev_abs[:, :29]
267
+ root_pos_delta = chunk_abs[:, 29:32] - prev_abs[:, 29:32]
268
+ root_quat_delta = quat_mul_wxyz(quat_inv_wxyz(prev_abs[:, 32:36]), chunk_abs[:, 32:36])
269
+ chunk_delta = np.concatenate([joint_delta, root_pos_delta, root_quat_delta], axis=1)
270
+
271
+ chunk_abs = normalize_quaternion_rows(chunk_abs.astype(np.float32))
272
+ if decode_textop_joint_from_delta:
273
+ chunk_pred_for_rtc = chunk_pred.copy()
274
+ chunk_pred_for_rtc[:, :29] = chunk_abs[:, :29]
275
+ prev_actions_norm = normalize_action_for_rtc(maxmin, chunk_pred_for_rtc)
276
+ else:
277
+ prev_actions_norm = pred_norm_clipped.copy()
278
+
279
+ take = min(exec_horizon, remaining_frames)
280
+ if take <= 0:
281
+ break
282
+ executed_abs = chunk_abs[:take]
283
+ executed_delta = chunk_delta[:take].astype(np.float32)
284
+ current_abs = executed_abs[-1].copy()
285
+ for frame_abs in executed_abs:
286
+ state_history.append(frame_abs.astype(np.float32).copy())
287
+ state_history = state_history[-state_horizon:]
288
+ all_abs.append(executed_abs)
289
+ all_delta.append(executed_delta)
290
+ remaining_frames -= take
291
+
292
+ pred_abs = np.concatenate(all_abs, axis=0)[:target_frames]
293
+ pred_delta = np.concatenate(all_delta, axis=0)[:target_frames]
294
+ pred_abs = normalize_quaternion_rows(pred_abs.astype(np.float32))
295
+ return pred_abs, pred_delta, num_chunks
296
+
297
+
298
+ def write_audio_info(
299
+ path: Path,
300
+ *,
301
+ audio_config_key: str,
302
+ source_motion_dir: Path,
303
+ video_path: Path,
304
+ initial_pose_source: str,
305
+ frames: int,
306
+ textop_actions: bool,
307
+ num_past_frames: int,
308
+ enable_rtc: bool,
309
+ action_exec_horizon: int,
310
+ num_inference_steps: int,
311
+ audio_sampling_rate: int,
312
+ audio_window_seconds: float,
313
+ full_audio: bool,
314
+ instruction: str,
315
+ ) -> None:
316
+ lines = [
317
+ "Motion Information: seed_audio_to_motion_inference",
318
+ "===================================================",
319
+ "",
320
+ f"audio_config_key: {audio_config_key}",
321
+ f"source_motion_dir: {source_motion_dir}",
322
+ f"video_path: {video_path}",
323
+ f"instruction: {instruction}",
324
+ f"initial_pose_source: {initial_pose_source}",
325
+ f"target_fps: {TARGET_FPS}",
326
+ f"num_inference_steps: {num_inference_steps}",
327
+ f"trained_with_textop_actions: {textop_actions}",
328
+ f"num_past_frames: {num_past_frames}",
329
+ f"rtc_enabled: {enable_rtc}",
330
+ f"action_exec_horizon: {action_exec_horizon}",
331
+ f"audio_sampling_rate: {audio_sampling_rate}",
332
+ f"full_audio: {full_audio}",
333
+ f"audio_window_seconds: {audio_window_seconds}",
334
+ "output_root_quaternion: wxyz",
335
+ "output_joint_angles: 29D",
336
+ "output_root_position: xyz",
337
+ "",
338
+ f"frames: {frames}",
339
+ "",
340
+ ]
341
+ path.write_text("\n".join(lines), encoding="utf-8")
342
+
343
+
344
+ def parse_args() -> argparse.Namespace:
345
+ parser = argparse.ArgumentParser(description="Seed audio->motion offline inference exporter")
346
+ parser.add_argument("--run-dir", type=str, required=True)
347
+ parser.add_argument("--ckpt-step", type=str, default="latest", help="integer step, 'latest', or 'best'")
348
+ parser.add_argument("--output-dir", type=str, required=True)
349
+ parser.add_argument("--motion-dir", type=str, required=True)
350
+ parser.add_argument("--video-path", type=str, default="")
351
+ parser.add_argument(
352
+ "--audio-config-key",
353
+ type=str,
354
+ default="audio",
355
+ choices=("audio", "audio_command"),
356
+ help="Which launch_config.data audio section to use",
357
+ )
358
+ parser.add_argument("--prompt", type=str, default="", help="Optional text instruction paired with the audio")
359
+ parser.add_argument("--num-output-frames", type=int, default=0, help="0 means source motion length")
360
+ parser.add_argument("--num-inference-steps", type=int, default=10)
361
+ parser.add_argument("--device", type=str, default="cuda:0")
362
+ parser.add_argument("--seed", type=int, default=42)
363
+ parser.add_argument("--export-delta", action="store_true", default=False)
364
+ parser.add_argument("--disable-rtc", action="store_true", default=False)
365
+ parser.add_argument("--no-align-start-hold", dest="align_start_hold", action="store_false")
366
+ parser.set_defaults(align_start_hold=True)
367
+ return parser.parse_args()
368
+
369
+
370
+ def main() -> None:
371
+ args = parse_args()
372
+
373
+ run_dir = Path(args.run_dir)
374
+ output_dir = Path(args.output_dir)
375
+ output_dir.mkdir(parents=True, exist_ok=True)
376
+
377
+ cfg_template: LaunchConfig = parse_args_to_tyro_config(run_dir / "argv.txt") # type: ignore
378
+ launch_config = cfg_template.model_validate_json((run_dir / "run_config.json").read_text())
379
+ seed_everything(launch_config.seed or args.seed)
380
+
381
+ ckpt_step = resolve_ckpt_step(run_dir, args.ckpt_step)
382
+ model = Psi0Model.from_pretrained(run_dir, ckpt_step, launch_config, device=args.device)
383
+ model.text_only = False
384
+ model.audio_only = True
385
+ model.video_only = False
386
+ model.to(args.device)
387
+ model.eval()
388
+
389
+ audio_cfg = audio_data_cfg(launch_config, args.audio_config_key)
390
+ data_root = Path(cfg_get(audio_cfg, "root_dir"))
391
+ video_root = Path(cfg_get(audio_cfg, "video_dir", ""))
392
+ motion_dir = resolve_motion_dir(data_root, args.motion_dir)
393
+ video_path = resolve_audio_path(
394
+ resolve_video_path(video_root, motion_dir, args.video_path),
395
+ str(cfg_get(audio_cfg, "audio_file_extensions", ".wav,.flac,.mp3,.m4a,.aac,.ogg")),
396
+ )
397
+ if not motion_dir.exists():
398
+ raise FileNotFoundError(f"motion_dir does not exist: {motion_dir}")
399
+ if not video_path.exists():
400
+ raise FileNotFoundError(f"video_path does not exist: {video_path}")
401
+
402
+ maxmin = launch_config.data.transform.field
403
+ odim = int(launch_config.model.odim)
404
+ textop_actions = bool(getattr(launch_config.data.transform.repack, "textop_actions", False))
405
+ decode_textop_joint_from_delta = textop_actions and should_decode_textop_joint_from_delta(launch_config)
406
+ num_past_frames = int(getattr(launch_config.data.transform.repack, "num_past_frames", 0))
407
+ use_future_progress_embedding = bool(getattr(launch_config.model, "use_future_progress_embedding", False))
408
+ enable_rtc = bool(getattr(launch_config.model, "rtc", False)) and (not args.disable_rtc)
409
+ action_exec_horizon = int(getattr(launch_config.model, "action_exec_horizon", int(model.action_horizon)))
410
+ rtc_max_delay = int(getattr(launch_config.model, "max_delay", 0))
411
+ audio_sampling_rate = int(cfg_get(audio_cfg, "audio_sampling_rate", 16000))
412
+ audio_window_seconds = float(cfg_get(audio_cfg, "audio_window_seconds", 1.0))
413
+ audio_left_context_seconds = float(cfg_get(audio_cfg, "audio_left_context_seconds", 0.0))
414
+ full_audio = bool(cfg_get(audio_cfg, "full_audio", False))
415
+
416
+ audio = load_audio_array(video_path, audio_sampling_rate)
417
+ instruction = args.prompt.strip()
418
+ x0 = load_pose_from_motion_dir(motion_dir)
419
+ source_frames = load_source_frame_count(motion_dir)
420
+ target_frames = int(args.num_output_frames) if args.num_output_frames > 0 else int(source_frames or 0)
421
+ start_hold_seconds = read_start_hold_seconds(motion_dir, bool(args.align_start_hold))
422
+
423
+ pred_abs, pred_delta, rollout_chunks = rollout_audio_motion(
424
+ model=model,
425
+ maxmin=maxmin,
426
+ odim=odim,
427
+ device=args.device,
428
+ instruction=instruction,
429
+ audio=audio,
430
+ audio_sampling_rate=audio_sampling_rate,
431
+ audio_window_seconds=audio_window_seconds,
432
+ audio_left_context_seconds=audio_left_context_seconds,
433
+ full_audio=full_audio,
434
+ start_hold_seconds=start_hold_seconds,
435
+ normalize_state=bool(getattr(maxmin, "normalize_state", False)),
436
+ textop_actions=textop_actions,
437
+ decode_textop_joint_from_delta=decode_textop_joint_from_delta,
438
+ num_past_frames=num_past_frames,
439
+ use_future_progress_embedding=use_future_progress_embedding,
440
+ enable_rtc=enable_rtc,
441
+ action_exec_horizon=action_exec_horizon,
442
+ rtc_max_delay=rtc_max_delay,
443
+ num_inference_steps=int(args.num_inference_steps),
444
+ target_frames=target_frames,
445
+ init_abs_state=x0,
446
+ )
447
+
448
+ motion_name = sanitize_motion_name(motion_dir.name)
449
+ sample_dir = output_dir / motion_name
450
+ sample_dir.mkdir(parents=True, exist_ok=True)
451
+
452
+ joint_pos = pred_abs[:, :29]
453
+ body_pos = pred_abs[:, 29:32]
454
+ body_quat = pred_abs[:, 32:36]
455
+
456
+ dt = 1.0 / float(TARGET_FPS)
457
+ joint_vel = finite_difference(joint_pos, dt)
458
+ body_lin_vel = finite_difference(body_pos, dt)
459
+ body_ang_vel = quaternion_angular_velocity(body_quat, dt)
460
+
461
+ write_csv(sample_dir / "joint_pos.csv", joint_pos, [f"joint_{i}" for i in range(29)])
462
+ write_csv(sample_dir / "joint_vel.csv", joint_vel, [f"joint_vel_{i}" for i in range(29)])
463
+ write_csv(sample_dir / "body_pos.csv", body_pos, ["body_0_x", "body_0_y", "body_0_z"])
464
+ write_csv(sample_dir / "body_quat.csv", body_quat, ["body_0_w", "body_0_x", "body_0_y", "body_0_z"])
465
+ write_csv(sample_dir / "body_lin_vel.csv", body_lin_vel, ["body_0_vel_x", "body_0_vel_y", "body_0_vel_z"])
466
+ write_csv(sample_dir / "body_ang_vel.csv", body_ang_vel, ["body_0_angvel_x", "body_0_angvel_y", "body_0_angvel_z"])
467
+ np.save(sample_dir / "motion.npy", pred_abs)
468
+
469
+ if args.export_delta:
470
+ write_csv(sample_dir / "joint_pos_delta.csv", pred_delta[:, :29], [f"joint_{i}" for i in range(29)])
471
+ write_csv(sample_dir / "body_pos_delta.csv", pred_delta[:, 29:32], ["body_0_x", "body_0_y", "body_0_z"])
472
+ write_csv(sample_dir / "body_quat_delta.csv", pred_delta[:, 32:36], ["body_0_w", "body_0_x", "body_0_y", "body_0_z"])
473
+ np.save(sample_dir / "motion_delta.npy", pred_delta)
474
+
475
+ write_metadata(sample_dir / "metadata.txt", motion_name, int(pred_abs.shape[0]))
476
+ write_audio_info(
477
+ sample_dir / "info.txt",
478
+ audio_config_key=str(args.audio_config_key),
479
+ source_motion_dir=motion_dir,
480
+ video_path=video_path,
481
+ initial_pose_source=str(motion_dir),
482
+ frames=int(pred_abs.shape[0]),
483
+ textop_actions=textop_actions,
484
+ num_past_frames=int(num_past_frames),
485
+ enable_rtc=enable_rtc,
486
+ action_exec_horizon=int(action_exec_horizon),
487
+ num_inference_steps=int(args.num_inference_steps),
488
+ audio_sampling_rate=int(audio_sampling_rate),
489
+ audio_window_seconds=float(audio_window_seconds),
490
+ full_audio=bool(full_audio),
491
+ instruction=instruction,
492
+ )
493
+
494
+ for src_name, dst_name in (("info.txt", "info_source.txt"), ("metadata.txt", "metadata_source.txt")):
495
+ src_path = motion_dir / src_name
496
+ if src_path.exists():
497
+ shutil.copy2(src_path, sample_dir / dst_name)
498
+
499
+ sample_meta = {
500
+ "run_dir": str(run_dir),
501
+ "ckpt_step": ckpt_step,
502
+ "audio_config_key": str(args.audio_config_key),
503
+ "instruction": instruction,
504
+ "source_motion_dir": str(motion_dir),
505
+ "video_path": str(video_path),
506
+ "output_dir": str(sample_dir),
507
+ "num_output_frames": int(pred_abs.shape[0]),
508
+ "rollout_chunks": int(rollout_chunks),
509
+ "textop_actions": textop_actions,
510
+ "decode_textop_joint_from_delta": decode_textop_joint_from_delta,
511
+ "num_past_frames": int(num_past_frames),
512
+ "enable_rtc": enable_rtc,
513
+ "action_exec_horizon": int(action_exec_horizon),
514
+ "audio_sampling_rate": int(audio_sampling_rate),
515
+ "full_audio": bool(full_audio),
516
+ "audio_window_seconds": float(audio_window_seconds),
517
+ "audio_left_context_seconds": float(audio_left_context_seconds),
518
+ "start_hold_seconds": float(start_hold_seconds),
519
+ }
520
+ (sample_dir / "infer_meta.json").write_text(json.dumps(sample_meta, ensure_ascii=False, indent=2), encoding="utf-8")
521
+ (output_dir / "manifest.json").write_text(
522
+ json.dumps({"samples": [sample_meta]}, ensure_ascii=False, indent=2),
523
+ encoding="utf-8",
524
+ )
525
+ print(f"[OK] audio sample -> {sample_dir}")
526
+ print(f"[DONE] wrote manifest: {output_dir / 'manifest.json'}")
527
+
528
+
529
+ if __name__ == "__main__":
530
+ main()
scripts/infer/seed_motion_teacher_reconstruct.py ADDED
@@ -0,0 +1,504 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/usr/bin/env python3
2
+ from __future__ import annotations
3
+
4
+ import argparse
5
+ import json
6
+ import math
7
+ import shutil
8
+ from pathlib import Path
9
+ from typing import List
10
+
11
+ import numpy as np
12
+ import pandas as pd
13
+ import torch
14
+ import torch.nn as nn
15
+ from safetensors import safe_open
16
+
17
+ from psi.config.config import LaunchConfig
18
+ from psi.data.seed.motion_repr import (
19
+ build_seed_textop_action,
20
+ quat_inv_wxyz,
21
+ quat_mul_wxyz,
22
+ recover_seed_abs_from_textop_action,
23
+ )
24
+ from psi.data.seed.progress import build_progress_conditions
25
+ from psi.models.motion_distill import MotionDistillPsi0Model
26
+ from psi.utils import parse_args_to_tyro_config, seed_everything
27
+
28
+ from seed_text_to_motion import (
29
+ TARGET_FPS,
30
+ build_model_states_from_abs,
31
+ finite_difference,
32
+ load_source_frame_count,
33
+ normalize_quaternion_rows,
34
+ quaternion_angular_velocity,
35
+ recover_seed_abs_from_textop_action_with_joint_delta,
36
+ resolve_ckpt_step,
37
+ sanitize_motion_name,
38
+ should_decode_textop_joint_from_delta,
39
+ write_csv,
40
+ write_metadata,
41
+ )
42
+
43
+
44
+ def resolve_checkpoint_dir(run_dir: Path, ckpt_step: int | str) -> Path:
45
+ if str(ckpt_step) in {"best", "best_ckpt"}:
46
+ ckpt_dir = run_dir / "checkpoints" / "best_ckpt"
47
+ else:
48
+ ckpt_dir = run_dir / "checkpoints" / f"ckpt_{ckpt_step}"
49
+ if not ckpt_dir.exists():
50
+ root_ckpt_dir = run_dir / f"ckpt_{ckpt_step}"
51
+ if root_ckpt_dir.exists():
52
+ ckpt_dir = root_ckpt_dir
53
+ if not ckpt_dir.exists():
54
+ raise FileNotFoundError(f"Checkpoint dir does not exist: {ckpt_dir}")
55
+ return ckpt_dir
56
+
57
+
58
+ def load_selected_checkpoint_state(ckpt_dir: Path, prefixes: tuple[str, ...]) -> dict[str, torch.Tensor]:
59
+ model_path = ckpt_dir / "model.safetensors"
60
+ index_path = ckpt_dir / "model.safetensors.index.json"
61
+ shard_paths: list[Path]
62
+
63
+ if index_path.exists():
64
+ weight_map = json.loads(index_path.read_text()).get("weight_map", {})
65
+ shard_paths = sorted(
66
+ {ckpt_dir / shard for key, shard in weight_map.items() if key.startswith(prefixes)}
67
+ )
68
+ elif model_path.exists():
69
+ shard_paths = [model_path]
70
+ else:
71
+ raise FileNotFoundError(f"No safetensors checkpoint found under {ckpt_dir}")
72
+
73
+ state_dict: dict[str, torch.Tensor] = {}
74
+ for shard_path in shard_paths:
75
+ with safe_open(str(shard_path), framework="pt", device="cpu") as f:
76
+ for key in f.keys():
77
+ if key.startswith(prefixes):
78
+ state_dict[key] = f.get_tensor(key)
79
+ return state_dict
80
+
81
+
82
+ def build_noise_scheduler(launch_config: LaunchConfig):
83
+ if launch_config.model.noise_scheduler == "ddpm":
84
+ from diffusers.schedulers.scheduling_ddim import DDIMScheduler
85
+
86
+ return DDIMScheduler(
87
+ num_train_timesteps=launch_config.model.train_diffusion_steps,
88
+ beta_start=0.0001,
89
+ beta_end=0.02,
90
+ beta_schedule="squaredcos_cap_v2",
91
+ clip_sample=True,
92
+ set_alpha_to_one=True,
93
+ steps_offset=0,
94
+ prediction_type="epsilon",
95
+ )
96
+ if launch_config.model.noise_scheduler == "flow":
97
+ from diffusers.schedulers.scheduling_flow_match_euler_discrete import FlowMatchEulerDiscreteScheduler
98
+
99
+ return FlowMatchEulerDiscreteScheduler(
100
+ num_train_timesteps=launch_config.model.train_diffusion_steps,
101
+ )
102
+ raise ValueError(f"Unsupported noise scheduler: {launch_config.model.noise_scheduler}")
103
+
104
+
105
+ def load_teacher_reconstruct_model(
106
+ run_dir: Path,
107
+ ckpt_step: int | str,
108
+ launch_config: LaunchConfig,
109
+ device: str,
110
+ ) -> MotionDistillPsi0Model:
111
+ ckpt_dir = resolve_checkpoint_dir(run_dir, ckpt_step)
112
+ state_dict = load_selected_checkpoint_state(
113
+ ckpt_dir,
114
+ prefixes=("action_header.", "motion_encoder."),
115
+ )
116
+ model = MotionDistillPsi0Model(
117
+ model_cfg=launch_config.model,
118
+ vlm_model=nn.Module(),
119
+ )
120
+ action_state = {
121
+ key[len("action_header.") :]: value
122
+ for key, value in state_dict.items()
123
+ if key.startswith("action_header.")
124
+ }
125
+ motion_state = {
126
+ key[len("motion_encoder.") :]: value
127
+ for key, value in state_dict.items()
128
+ if key.startswith("motion_encoder.")
129
+ }
130
+ if not action_state:
131
+ raise ValueError(f"No action_header weights found in {ckpt_dir}")
132
+ if not motion_state:
133
+ raise ValueError(f"No motion_encoder weights found in {ckpt_dir}")
134
+ model.action_header.load_state_dict(action_state, strict=True)
135
+ model.motion_encoder.load_state_dict(motion_state, strict=True)
136
+ model.noise_scheduler = build_noise_scheduler(launch_config)
137
+ model.action_horizon = int(launch_config.model.action_chunk_size)
138
+ model.action_dim = int(launch_config.model.action_dim)
139
+ model.device = device
140
+ return model.to(device)
141
+
142
+
143
+ def load_motion_abs(motion_dir: Path) -> np.ndarray:
144
+ joint = pd.read_csv(motion_dir / "joint_pos.csv").to_numpy(dtype=np.float32)
145
+ body_pos = pd.read_csv(motion_dir / "body_pos.csv").to_numpy(dtype=np.float32)
146
+ body_quat = pd.read_csv(motion_dir / "body_quat.csv").to_numpy(dtype=np.float32)
147
+ if not (len(joint) == len(body_pos) == len(body_quat)):
148
+ raise ValueError(f"Motion csv length mismatch under {motion_dir}")
149
+ return np.concatenate([joint, body_pos, body_quat], axis=1).astype(np.float32)
150
+
151
+
152
+ def build_abs_action_chunk(
153
+ motion_abs: np.ndarray,
154
+ frame_id: int,
155
+ horizon: int,
156
+ sample_stride: int = 1,
157
+ ) -> np.ndarray:
158
+ total_frames = int(motion_abs.shape[0])
159
+ actions: List[np.ndarray] = []
160
+ for step in range(horizon):
161
+ raw_index = frame_id + (step + 1) * sample_stride
162
+ actions.append(motion_abs[min(raw_index, total_frames - 1)])
163
+ return np.stack(actions, axis=0).astype(np.float32)
164
+
165
+
166
+ def build_teacher_action_features(
167
+ actions_abs: np.ndarray,
168
+ current_abs: np.ndarray,
169
+ *,
170
+ textop_actions: bool,
171
+ action_dim: int,
172
+ ) -> np.ndarray:
173
+ if textop_actions:
174
+ actions = build_seed_textop_action(actions_abs[:, :36], current_abs[:36])
175
+ else:
176
+ actions = actions_abs[:, :action_dim]
177
+
178
+ if actions.shape[-1] < action_dim:
179
+ pad = np.zeros((actions.shape[0], action_dim - actions.shape[-1]), dtype=np.float32)
180
+ actions = np.concatenate([actions, pad], axis=-1)
181
+ if actions.shape[-1] != action_dim:
182
+ raise ValueError(f"Teacher action dim mismatch: built {actions.shape[-1]}, expected {action_dim}")
183
+ return actions.astype(np.float32)
184
+
185
+
186
+ def normalize_action_only(maxmin, actions: np.ndarray) -> np.ndarray:
187
+ action_min = np.asarray(maxmin.action_min, dtype=np.float32)
188
+ action_max = np.asarray(maxmin.action_max, dtype=np.float32)
189
+ ill_mask = np.abs(action_max - action_min) < 1e-4 * (
190
+ np.abs(action_max) + np.abs(action_min) + 1e-8
191
+ )
192
+ denom = action_max - action_min
193
+ denom[ill_mask] = 1.0
194
+ actions_norm = np.where(
195
+ ill_mask,
196
+ actions,
197
+ (actions - action_min) / denom * 2 - 1,
198
+ )
199
+ if bool(getattr(maxmin, "use_norm_mask", False)):
200
+ action_norm_masks = np.asarray(maxmin.action_norm_masks, dtype=bool)
201
+ actions_norm = np.where(action_norm_masks, actions_norm, actions)
202
+ return np.clip(actions_norm, -1.0, 1.0).astype(np.float32)
203
+
204
+
205
+ def decode_action_chunk(
206
+ chunk_pred: np.ndarray,
207
+ current_abs: np.ndarray,
208
+ *,
209
+ textop_actions: bool,
210
+ decode_textop_joint_from_delta: bool,
211
+ ) -> tuple[np.ndarray, np.ndarray]:
212
+ if textop_actions:
213
+ if decode_textop_joint_from_delta:
214
+ chunk_abs = recover_seed_abs_from_textop_action_with_joint_delta(chunk_pred, current_abs)
215
+ else:
216
+ chunk_abs = recover_seed_abs_from_textop_action(chunk_pred, current_abs)
217
+ prev_abs = np.concatenate([current_abs[None, :], chunk_abs[:-1]], axis=0)
218
+ joint_delta = chunk_abs[:, :29] - prev_abs[:, :29]
219
+ root_pos_delta = chunk_abs[:, 29:32] - prev_abs[:, 29:32]
220
+ root_quat_delta = quat_mul_wxyz(quat_inv_wxyz(prev_abs[:, 32:36]), chunk_abs[:, 32:36])
221
+ chunk_delta = np.concatenate([joint_delta, root_pos_delta, root_quat_delta], axis=1)
222
+ else:
223
+ chunk_abs = chunk_pred[:, :36]
224
+ first_delta = chunk_abs[0] - current_abs
225
+ if chunk_abs.shape[0] > 1:
226
+ rest_delta = chunk_abs[1:] - chunk_abs[:-1]
227
+ chunk_delta = np.concatenate([first_delta[None, :], rest_delta], axis=0)
228
+ else:
229
+ chunk_delta = first_delta[None, :]
230
+
231
+ return normalize_quaternion_rows(chunk_abs.astype(np.float32)), chunk_delta.astype(np.float32)
232
+
233
+
234
+ def reconstruct_teacher_motion(
235
+ *,
236
+ model: MotionDistillPsi0Model,
237
+ maxmin,
238
+ motion_abs: np.ndarray,
239
+ odim: int,
240
+ action_dim: int,
241
+ device: str,
242
+ normalize_state: bool,
243
+ textop_actions: bool,
244
+ decode_textop_joint_from_delta: bool,
245
+ num_past_frames: int,
246
+ use_future_progress_embedding: bool,
247
+ num_inference_steps: int,
248
+ target_frames: int,
249
+ ) -> tuple[np.ndarray, np.ndarray, dict[str, float]]:
250
+ horizon = int(model.action_horizon)
251
+ if target_frames <= 0:
252
+ target_frames = max(1, int(motion_abs.shape[0]) - 1)
253
+
254
+ num_chunks = int(math.ceil(float(target_frames) / float(horizon)))
255
+ all_abs: List[np.ndarray] = []
256
+ all_delta: List[np.ndarray] = []
257
+ remaining_frames = int(target_frames)
258
+
259
+ for chunk_id in range(num_chunks):
260
+ frame_id = chunk_id * horizon
261
+ current_abs = motion_abs[min(frame_id, motion_abs.shape[0] - 1)].astype(np.float32)
262
+ state_indices = [
263
+ max(0, frame_id - (int(num_past_frames) - t))
264
+ for t in range(int(num_past_frames) + 1)
265
+ ]
266
+ states = build_model_states_from_abs(
267
+ [motion_abs[min(i, motion_abs.shape[0] - 1)] for i in state_indices],
268
+ textop_actions=textop_actions,
269
+ odim=odim,
270
+ )
271
+ if normalize_state:
272
+ states = maxmin.normalize_state_func(states)
273
+
274
+ actions_abs = build_abs_action_chunk(motion_abs, frame_id, horizon)
275
+ teacher_actions = build_teacher_action_features(
276
+ actions_abs,
277
+ current_abs,
278
+ textop_actions=textop_actions,
279
+ action_dim=action_dim,
280
+ )
281
+ teacher_actions_norm = normalize_action_only(maxmin, teacher_actions)
282
+
283
+ with torch.inference_mode():
284
+ states_t = torch.from_numpy(states).unsqueeze(0).to(device)
285
+ teacher_actions_t = torch.from_numpy(teacher_actions_norm).unsqueeze(0).to(device)
286
+ hidden_states = model.encode_motion_hidden(teacher_actions_t)
287
+
288
+ future_progress_t = None
289
+ if use_future_progress_embedding:
290
+ _, future_progress = build_progress_conditions(
291
+ frame_id=frame_id,
292
+ total_frames=int(motion_abs.shape[0]),
293
+ action_horizon=horizon,
294
+ )
295
+ future_progress_t = torch.from_numpy(future_progress[None]).to(device)
296
+
297
+ action_samples = torch.randn(1, horizon, action_dim, device=device)
298
+ model.noise_scheduler.set_timesteps(num_inference_steps)
299
+ for timestep in model.noise_scheduler.timesteps:
300
+ batched_timestep = timestep.expand(1).to(device)
301
+ model_output = model.action_from_hidden(
302
+ hidden_states=hidden_states,
303
+ attention_mask=None,
304
+ action_samples=action_samples,
305
+ states=states_t,
306
+ timestep=batched_timestep,
307
+ traj2ds=None,
308
+ future_progress=future_progress_t,
309
+ return_dict=True,
310
+ )
311
+ action_samples = model.noise_scheduler.step(
312
+ model_output=model_output.action,
313
+ timestep=timestep,
314
+ sample=action_samples,
315
+ ).prev_sample
316
+
317
+ pred_norm = action_samples[0].float().cpu().numpy()
318
+ pred_norm_clipped = np.clip(np.asarray(pred_norm, dtype=np.float32), -1.0, 1.0)
319
+ chunk_pred = np.asarray(maxmin.denormalize(pred_norm_clipped), dtype=np.float32)
320
+ chunk_abs, chunk_delta = decode_action_chunk(
321
+ chunk_pred,
322
+ current_abs,
323
+ textop_actions=textop_actions,
324
+ decode_textop_joint_from_delta=decode_textop_joint_from_delta,
325
+ )
326
+
327
+ take = min(horizon, remaining_frames)
328
+ all_abs.append(chunk_abs[:take])
329
+ all_delta.append(chunk_delta[:take])
330
+ remaining_frames -= take
331
+
332
+ pred_abs = np.concatenate(all_abs, axis=0)[:target_frames]
333
+ pred_delta = np.concatenate(all_delta, axis=0)[:target_frames]
334
+ pred_abs = normalize_quaternion_rows(pred_abs.astype(np.float32))
335
+
336
+ ref_abs = motion_abs[1 : 1 + pred_abs.shape[0], :36]
337
+ if ref_abs.shape[0] == pred_abs.shape[0]:
338
+ diff = pred_abs - ref_abs
339
+ metrics = {
340
+ "joint_mse": float(np.mean(diff[:, :29] ** 2)),
341
+ "root_pos_mse": float(np.mean(diff[:, 29:32] ** 2)),
342
+ "all_abs_mse": float(np.mean(diff ** 2)),
343
+ }
344
+ else:
345
+ metrics = {}
346
+ return pred_abs, pred_delta, metrics
347
+
348
+
349
+ def write_teacher_info(
350
+ path: Path,
351
+ *,
352
+ source_motion_dir: Path,
353
+ frames: int,
354
+ textop_actions: bool,
355
+ num_past_frames: int,
356
+ num_inference_steps: int,
357
+ metrics: dict[str, float],
358
+ ) -> None:
359
+ lines = [
360
+ "Motion Information: motion_distill_teacher_reconstruct",
361
+ "======================================================",
362
+ "",
363
+ f"source_motion_dir: {source_motion_dir}",
364
+ f"target_fps: {TARGET_FPS}",
365
+ f"num_inference_steps: {num_inference_steps}",
366
+ f"trained_with_textop_actions: {textop_actions}",
367
+ f"num_past_frames: {num_past_frames}",
368
+ "condition: motion_encoder(ground_truth_actions)",
369
+ "output_root_quaternion: wxyz",
370
+ "output_joint_angles: 29D",
371
+ "output_root_position: xyz",
372
+ "",
373
+ f"frames: {frames}",
374
+ "",
375
+ ]
376
+ for key, value in metrics.items():
377
+ lines.append(f"{key}: {value:.8f}")
378
+ path.write_text("\n".join(lines), encoding="utf-8")
379
+
380
+
381
+ def parse_args() -> argparse.Namespace:
382
+ parser = argparse.ArgumentParser(description="Motion-distill teacher motion reconstruction exporter")
383
+ parser.add_argument("--run-dir", type=str, required=True)
384
+ parser.add_argument("--ckpt-step", type=str, default="best", help="integer step, 'latest', or 'best'")
385
+ parser.add_argument("--output-dir", type=str, required=True)
386
+ parser.add_argument("--motion-dir", type=str, required=True)
387
+ parser.add_argument("--num-output-frames", type=int, default=0, help="0 means source motion length minus one")
388
+ parser.add_argument("--num-inference-steps", type=int, default=10)
389
+ parser.add_argument("--device", type=str, default="cuda:0")
390
+ parser.add_argument("--seed", type=int, default=42)
391
+ parser.add_argument("--export-delta", action="store_true", default=False)
392
+ return parser.parse_args()
393
+
394
+
395
+ def main() -> None:
396
+ args = parse_args()
397
+ run_dir = Path(args.run_dir)
398
+ output_dir = Path(args.output_dir)
399
+ motion_dir = Path(args.motion_dir)
400
+ output_dir.mkdir(parents=True, exist_ok=True)
401
+
402
+ cfg_template: LaunchConfig = parse_args_to_tyro_config(run_dir / "argv.txt") # type: ignore
403
+ launch_config = cfg_template.model_validate_json((run_dir / "run_config.json").read_text())
404
+ seed_everything(launch_config.seed or args.seed)
405
+
406
+ ckpt_step = resolve_ckpt_step(run_dir, args.ckpt_step)
407
+ model = load_teacher_reconstruct_model(run_dir, ckpt_step, launch_config, device=args.device)
408
+ model.eval()
409
+
410
+ motion_abs = load_motion_abs(motion_dir)
411
+ source_frames = load_source_frame_count(motion_dir) or int(motion_abs.shape[0])
412
+ target_frames = int(args.num_output_frames) if args.num_output_frames > 0 else max(1, source_frames - 1)
413
+
414
+ maxmin = launch_config.data.transform.field
415
+ odim = int(launch_config.model.odim)
416
+ action_dim = int(getattr(launch_config.model, "action_dim", getattr(model, "action_dim", 39)))
417
+ textop_actions = bool(getattr(launch_config.data.transform.repack, "textop_actions", False))
418
+ decode_textop_joint_from_delta = textop_actions and should_decode_textop_joint_from_delta(launch_config)
419
+ num_past_frames = int(getattr(launch_config.data.transform.repack, "num_past_frames", 0))
420
+ use_future_progress_embedding = bool(getattr(launch_config.model, "use_future_progress_embedding", False))
421
+
422
+ pred_abs, pred_delta, metrics = reconstruct_teacher_motion(
423
+ model=model,
424
+ maxmin=maxmin,
425
+ motion_abs=motion_abs,
426
+ odim=odim,
427
+ action_dim=action_dim,
428
+ device=args.device,
429
+ normalize_state=bool(getattr(maxmin, "normalize_state", False)),
430
+ textop_actions=textop_actions,
431
+ decode_textop_joint_from_delta=decode_textop_joint_from_delta,
432
+ num_past_frames=num_past_frames,
433
+ use_future_progress_embedding=use_future_progress_embedding,
434
+ num_inference_steps=int(args.num_inference_steps),
435
+ target_frames=target_frames,
436
+ )
437
+
438
+ motion_name = sanitize_motion_name(motion_dir.name)
439
+ sample_dir = output_dir / motion_name
440
+ sample_dir.mkdir(parents=True, exist_ok=True)
441
+
442
+ joint_pos = pred_abs[:, :29]
443
+ body_pos = pred_abs[:, 29:32]
444
+ body_quat = pred_abs[:, 32:36]
445
+
446
+ dt = 1.0 / float(TARGET_FPS)
447
+ joint_vel = finite_difference(joint_pos, dt)
448
+ body_lin_vel = finite_difference(body_pos, dt)
449
+ body_ang_vel = quaternion_angular_velocity(body_quat, dt)
450
+
451
+ write_csv(sample_dir / "joint_pos.csv", joint_pos, [f"joint_{i}" for i in range(29)])
452
+ write_csv(sample_dir / "joint_vel.csv", joint_vel, [f"joint_vel_{i}" for i in range(29)])
453
+ write_csv(sample_dir / "body_pos.csv", body_pos, ["body_0_x", "body_0_y", "body_0_z"])
454
+ write_csv(sample_dir / "body_quat.csv", body_quat, ["body_0_w", "body_0_x", "body_0_y", "body_0_z"])
455
+ write_csv(sample_dir / "body_lin_vel.csv", body_lin_vel, ["body_0_vel_x", "body_0_vel_y", "body_0_vel_z"])
456
+ write_csv(sample_dir / "body_ang_vel.csv", body_ang_vel, ["body_0_angvel_x", "body_0_angvel_y", "body_0_angvel_z"])
457
+ np.save(sample_dir / "motion.npy", pred_abs)
458
+
459
+ if args.export_delta:
460
+ write_csv(sample_dir / "joint_pos_delta.csv", pred_delta[:, :29], [f"joint_{i}" for i in range(29)])
461
+ write_csv(sample_dir / "body_pos_delta.csv", pred_delta[:, 29:32], ["body_0_x", "body_0_y", "body_0_z"])
462
+ write_csv(sample_dir / "body_quat_delta.csv", pred_delta[:, 32:36], ["body_0_w", "body_0_x", "body_0_y", "body_0_z"])
463
+ np.save(sample_dir / "motion_delta.npy", pred_delta)
464
+
465
+ write_metadata(sample_dir / "metadata.txt", motion_name, int(pred_abs.shape[0]))
466
+ write_teacher_info(
467
+ sample_dir / "info.txt",
468
+ source_motion_dir=motion_dir,
469
+ frames=int(pred_abs.shape[0]),
470
+ textop_actions=textop_actions,
471
+ num_past_frames=num_past_frames,
472
+ num_inference_steps=int(args.num_inference_steps),
473
+ metrics=metrics,
474
+ )
475
+
476
+ for src_name, dst_name in (("info.txt", "info_source.txt"), ("metadata.txt", "metadata_source.txt")):
477
+ src_path = motion_dir / src_name
478
+ if src_path.exists():
479
+ shutil.copy2(src_path, sample_dir / dst_name)
480
+
481
+ sample_meta = {
482
+ "run_dir": str(run_dir),
483
+ "ckpt_step": ckpt_step,
484
+ "source_motion_dir": str(motion_dir),
485
+ "output_dir": str(sample_dir),
486
+ "num_output_frames": int(pred_abs.shape[0]),
487
+ "textop_actions": textop_actions,
488
+ "decode_textop_joint_from_delta": decode_textop_joint_from_delta,
489
+ "num_past_frames": int(num_past_frames),
490
+ "num_inference_steps": int(args.num_inference_steps),
491
+ "metrics": metrics,
492
+ }
493
+ (sample_dir / "infer_meta.json").write_text(json.dumps(sample_meta, ensure_ascii=False, indent=2), encoding="utf-8")
494
+ (output_dir / "manifest.json").write_text(
495
+ json.dumps({"samples": [sample_meta]}, ensure_ascii=False, indent=2),
496
+ encoding="utf-8",
497
+ )
498
+ print(f"[OK] teacher reconstruction -> {sample_dir}")
499
+ if metrics:
500
+ print("[METRICS] " + json.dumps(metrics, ensure_ascii=False))
501
+
502
+
503
+ if __name__ == "__main__":
504
+ main()
scripts/infer/seed_text_to_motion.py ADDED
@@ -0,0 +1,873 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/usr/bin/env python3
2
+ from __future__ import annotations
3
+
4
+ import argparse
5
+ import json
6
+ import math
7
+ import shutil
8
+ from pathlib import Path
9
+ from typing import List
10
+
11
+ import numpy as np
12
+ import pandas as pd
13
+ import torch
14
+
15
+ from psi.config.config import LaunchConfig
16
+ from psi.data.seed.motion_repr import (
17
+ SEED_ABS_DIM,
18
+ SEED_TEXTOP_ACTION_DIM,
19
+ TEXTOP_JOINT_DELTA,
20
+ TEXTOP_ROOT_DELTA_LOCAL,
21
+ TEXTOP_ROOT_HEIGHT,
22
+ TEXTOP_ROOT_ROT6D_DELTA,
23
+ quat_inv_wxyz,
24
+ quat_mul_wxyz,
25
+ build_seed_textop_state,
26
+ recover_seed_abs_from_textop_action,
27
+ rot6d_delta_to_quat,
28
+ world_delta_from_local,
29
+ )
30
+ from psi.data.seed.progress import build_progress_conditions
31
+ from psi.data.seed.text_prompt import (
32
+ DEFAULT_SEED_TEXT_FIELDS,
33
+ compose_seed_motion_prompt,
34
+ compose_seed_motion_prompt_from_row,
35
+ )
36
+ from psi.models.psi0 import Psi0Model
37
+ from psi.utils import parse_args_to_tyro_config, seed_everything
38
+
39
+
40
+ DEFAULT_TEXT_FIELDS: List[str] = list(DEFAULT_SEED_TEXT_FIELDS)
41
+
42
+ TARGET_FPS = 50
43
+
44
+
45
+ def compose_instruction(row: pd.Series, text_fields: List[str]) -> str:
46
+ return compose_seed_motion_prompt_from_row(row, text_fields)
47
+
48
+
49
+ def map_motion_dir(root_dir: Path, move_g1_path: str) -> Path:
50
+ rel = str(move_g1_path).strip().replace("\\", "/")
51
+ rel = rel.replace("g1/csv/", "g1/csv_sonic/")
52
+ rel = rel.replace("/csv/", "/csv_sonic/")
53
+ if rel.endswith(".csv"):
54
+ rel = rel[:-4]
55
+
56
+ if rel.startswith("dataset/seed/"):
57
+ rel = rel[len("dataset/seed/") :]
58
+ if rel.startswith("seed/"):
59
+ rel = rel[len("seed/") :]
60
+
61
+ p = Path(rel)
62
+ if p.is_absolute():
63
+ return p
64
+ return root_dir / p
65
+
66
+
67
+ def sanitize_motion_name(name: str) -> str:
68
+ out = "".join(ch if (ch.isalnum() or ch in {"_", "-", "."}) else "_" for ch in name.strip())
69
+ out = out.strip("._")
70
+ return out or "manual_prompt"
71
+
72
+
73
+ def normalize_quaternion_rows(motion_abs: np.ndarray) -> np.ndarray:
74
+ if motion_abs.shape[1] < 36:
75
+ return motion_abs
76
+ q = motion_abs[:, 32:36]
77
+ norm = np.linalg.norm(q, axis=1, keepdims=True)
78
+ norm = np.where(norm < 1e-8, 1.0, norm)
79
+ motion_abs[:, 32:36] = q / norm
80
+ return motion_abs
81
+
82
+
83
+ def quat_multiply(q1: np.ndarray, q2: np.ndarray) -> np.ndarray:
84
+ w1, x1, y1, z1 = q1
85
+ w2, x2, y2, z2 = q2
86
+ return np.array(
87
+ [
88
+ w1 * w2 - x1 * x2 - y1 * y2 - z1 * z2,
89
+ w1 * x2 + x1 * w2 + y1 * z2 - z1 * y2,
90
+ w1 * y2 - x1 * z2 + y1 * w2 + z1 * x2,
91
+ w1 * z2 + x1 * y2 - y1 * x2 + z1 * w2,
92
+ ],
93
+ dtype=np.float64,
94
+ )
95
+
96
+
97
+ def quat_conjugate(q: np.ndarray) -> np.ndarray:
98
+ return np.array([q[0], -q[1], -q[2], -q[3]], dtype=np.float64)
99
+
100
+
101
+ def quat_to_rotvec(q: np.ndarray) -> np.ndarray:
102
+ q = q / max(np.linalg.norm(q), 1e-12)
103
+ if q[0] < 0.0:
104
+ q = -q
105
+ vec = q[1:]
106
+ sin_half = np.linalg.norm(vec)
107
+ if sin_half < 1e-10:
108
+ return np.zeros(3, dtype=np.float64)
109
+ axis = vec / sin_half
110
+ angle = 2.0 * np.arctan2(sin_half, q[0])
111
+ return axis * angle
112
+
113
+
114
+ def finite_difference(data: np.ndarray, dt: float) -> np.ndarray:
115
+ if data.shape[0] < 2:
116
+ return np.zeros_like(data, dtype=np.float64)
117
+ edge_order = 2 if data.shape[0] > 2 else 1
118
+ return np.gradient(data, dt, axis=0, edge_order=edge_order)
119
+
120
+
121
+ def quaternion_angular_velocity(quat_wxyz: np.ndarray, dt: float) -> np.ndarray:
122
+ ang_vel = np.zeros((quat_wxyz.shape[0], 3), dtype=np.float64)
123
+ if quat_wxyz.shape[0] < 2:
124
+ return ang_vel
125
+
126
+ for i in range(quat_wxyz.shape[0] - 1):
127
+ dq = quat_multiply(quat_wxyz[i + 1], quat_conjugate(quat_wxyz[i]))
128
+ ang_vel[i] = quat_to_rotvec(dq) / dt
129
+
130
+ ang_vel[-1] = ang_vel[-2]
131
+ return ang_vel
132
+
133
+
134
+ def write_csv(path: Path, array: np.ndarray, headers: List[str]) -> None:
135
+ np.savetxt(
136
+ path,
137
+ np.asarray(array, dtype=np.float64),
138
+ delimiter=",",
139
+ header=",".join(headers),
140
+ comments="",
141
+ fmt="%.6f",
142
+ )
143
+
144
+
145
+ def write_metadata(path: Path, motion_name: str, timesteps: int) -> None:
146
+ path.write_text(
147
+ "\n".join(
148
+ [
149
+ f"Metadata for: {motion_name}",
150
+ "==============================",
151
+ "",
152
+ "Body part indexes:",
153
+ "[0]",
154
+ "",
155
+ f"Total timesteps: {timesteps}",
156
+ "",
157
+ ]
158
+ ),
159
+ encoding="utf-8",
160
+ )
161
+
162
+
163
+ def write_info(
164
+ path: Path,
165
+ *,
166
+ source_path: str,
167
+ initial_pose_source: str,
168
+ frames: int,
169
+ use_delta_actions: bool,
170
+ root_delta_actions: bool,
171
+ textop_actions: bool,
172
+ num_past_frames: int,
173
+ enable_rtc: bool,
174
+ action_exec_horizon: int,
175
+ num_inference_steps: int,
176
+ ) -> None:
177
+ lines = [
178
+ "Motion Information: seed_text_to_motion_inference",
179
+ "==================================================",
180
+ "",
181
+ f"source_path: {source_path}",
182
+ f"initial_pose_source: {initial_pose_source}",
183
+ f"target_fps: {TARGET_FPS}",
184
+ f"num_inference_steps: {num_inference_steps}",
185
+ f"trained_with_delta_actions: {use_delta_actions}",
186
+ f"trained_with_root_delta_actions: {root_delta_actions}",
187
+ f"trained_with_textop_actions: {textop_actions}",
188
+ f"num_past_frames: {num_past_frames}",
189
+ f"rtc_enabled: {enable_rtc}",
190
+ f"action_exec_horizon: {action_exec_horizon}",
191
+ "output_root_quaternion: wxyz",
192
+ "output_joint_angles: 29D",
193
+ "output_root_position: xyz",
194
+ "",
195
+ f"frames: {frames}",
196
+ "",
197
+ ]
198
+ path.write_text("\n".join(lines), encoding="utf-8")
199
+
200
+
201
+ def load_pose_from_motion_dir(motion_dir: Path) -> np.ndarray:
202
+ joint = pd.read_csv(motion_dir / "joint_pos.csv").to_numpy(dtype=np.float32)
203
+ body_pos = pd.read_csv(motion_dir / "body_pos.csv").to_numpy(dtype=np.float32)
204
+ body_quat = pd.read_csv(motion_dir / "body_quat.csv").to_numpy(dtype=np.float32)
205
+ return np.concatenate([joint[0], body_pos[0], body_quat[0]], axis=0).astype(np.float32)
206
+
207
+
208
+ def load_pose_from_npy(path: Path) -> np.ndarray:
209
+ pose = np.load(path).astype(np.float32)
210
+ if pose.ndim == 2:
211
+ pose = pose[0]
212
+ if pose.shape != (36,):
213
+ raise ValueError(f"Expected init pose shape (36,) or (T,36), got {pose.shape} from {path}")
214
+ return pose.astype(np.float32)
215
+
216
+
217
+ def resolve_motion_dir(data_root: Path, motion_dir_arg: str) -> Path:
218
+ motion_dir = Path(motion_dir_arg).expanduser()
219
+ if motion_dir.is_absolute():
220
+ return motion_dir
221
+ return data_root / motion_dir
222
+
223
+
224
+ def load_initial_pose(root_dir: Path, row: pd.Series, restore_from_data: bool, strict: bool = False) -> np.ndarray:
225
+ if not restore_from_data:
226
+ if strict:
227
+ # <modify_jerk> root-delta也必须有真实首帧,否则root轨迹会从zero累加跑偏。
228
+ raise ValueError("Delta/root-delta inference requires a real initial pose. Do not use --no-restore-from-data.")
229
+ return np.zeros((36,), dtype=np.float32)
230
+
231
+ try:
232
+ motion_dir = map_motion_dir(root_dir, str(row["move_g1_path"]))
233
+ return load_pose_from_motion_dir(motion_dir)
234
+ except Exception as exc:
235
+ if strict:
236
+ raise RuntimeError(f"Failed to load initial pose from metadata row: {exc}") from exc
237
+ return np.zeros((36,), dtype=np.float32)
238
+
239
+
240
+ def load_source_frame_count(source_motion_dir: Path) -> int | None:
241
+ try:
242
+ joint = pd.read_csv(source_motion_dir / "joint_pos.csv")
243
+ frames = int(len(joint))
244
+ if frames > 0:
245
+ return frames
246
+ except Exception:
247
+ pass
248
+ return None
249
+
250
+
251
+ def build_model_states_from_abs(
252
+ state_history_abs: List[np.ndarray],
253
+ *,
254
+ textop_actions: bool,
255
+ odim: int,
256
+ ) -> np.ndarray:
257
+ states_abs = np.stack(state_history_abs, axis=0).astype(np.float32)
258
+ states = build_seed_textop_state(states_abs[:, :SEED_ABS_DIM]) if textop_actions else states_abs
259
+ if states.shape[-1] < odim:
260
+ pad = np.zeros((states.shape[0], odim - states.shape[-1]), dtype=np.float32)
261
+ states = np.concatenate([states, pad], axis=-1)
262
+ if states.shape[-1] != odim:
263
+ raise ValueError(f"Infer state dim mismatch: built {states.shape[-1]}D, model odim={odim}")
264
+ return states.astype(np.float32)
265
+
266
+
267
+ def should_decode_textop_joint_from_delta(launch_config: LaunchConfig) -> bool:
268
+ if SEED_TEXTOP_ACTION_DIM == 39:
269
+ return False
270
+ group_weight = getattr(getattr(launch_config, "train", None), "loss_group_weight", None)
271
+ if group_weight is None:
272
+ return False
273
+ joint_abs_weight = float(getattr(group_weight, "joint_abs", 1.0))
274
+ joint_delta_weight = float(getattr(group_weight, "joint_delta", 1.0))
275
+ return joint_abs_weight == 0.0 and joint_delta_weight != 0.0
276
+
277
+
278
+ def recover_seed_abs_from_textop_action_with_joint_delta(
279
+ features: np.ndarray,
280
+ current_abs: np.ndarray,
281
+ ) -> np.ndarray:
282
+ # breakpoint()
283
+ # 当joint_abs不参与训练时,推理侧也改成用joint_delta递推关节,避免使用未监督的joint_abs通道。
284
+ features = np.asarray(features, dtype=np.float32)
285
+ current_abs = np.asarray(current_abs, dtype=np.float32).copy()
286
+ if features.shape[-1] != SEED_TEXTOP_ACTION_DIM:
287
+ raise ValueError(f"Expected textop action dim {SEED_TEXTOP_ACTION_DIM}, got {features.shape[-1]}")
288
+ if current_abs.shape[-1] != SEED_ABS_DIM:
289
+ raise ValueError(f"Expected current_abs dim {SEED_ABS_DIM}, got {current_abs.shape[-1]}")
290
+ if TEXTOP_JOINT_DELTA.stop == TEXTOP_JOINT_DELTA.start:
291
+ return recover_seed_abs_from_textop_action(features, current_abs)
292
+
293
+ frames = []
294
+ prev_abs = current_abs.copy()
295
+ for feat in features:
296
+ joint_delta = feat[TEXTOP_JOINT_DELTA]
297
+ root_delta_local = feat[TEXTOP_ROOT_DELTA_LOCAL]
298
+ root_rot_delta = feat[TEXTOP_ROOT_ROT6D_DELTA]
299
+ root_height = feat[TEXTOP_ROOT_HEIGHT][0]
300
+
301
+ joint_abs = prev_abs[:29] + joint_delta
302
+ root_pos = prev_abs[29:32] + world_delta_from_local(prev_abs[32:36], root_delta_local)
303
+ root_pos[2] = root_height
304
+ rel_quat = rot6d_delta_to_quat(root_rot_delta)
305
+ root_quat = quat_mul_wxyz(prev_abs[32:36], rel_quat)
306
+
307
+ frame = np.concatenate([joint_abs, root_pos.astype(np.float32), root_quat.astype(np.float32)], axis=0)
308
+ frame = frame.astype(np.float32)
309
+ frames.append(frame)
310
+ prev_abs = frame
311
+
312
+ return np.stack(frames, axis=0).astype(np.float32)
313
+
314
+
315
+ def resolve_metadata_csv(data_root: Path, launch_config: LaunchConfig, metadata_csv_arg: str) -> Path | None:
316
+ if metadata_csv_arg:
317
+ metadata_csv = Path(metadata_csv_arg)
318
+ elif hasattr(launch_config.data, "metadata_csv"):
319
+ metadata_csv = Path(launch_config.data.metadata_csv)
320
+ elif hasattr(launch_config.data, "text"):
321
+ text_cfg = launch_config.data.text
322
+ metadata_csv_value = text_cfg.get("metadata_csv") if isinstance(text_cfg, dict) else getattr(text_cfg, "metadata_csv", None)
323
+ if metadata_csv_value is None:
324
+ return None
325
+ metadata_csv = Path(metadata_csv_value)
326
+ else:
327
+ return None
328
+ if not metadata_csv.is_absolute():
329
+ metadata_csv = data_root / metadata_csv
330
+ return metadata_csv
331
+
332
+
333
+ def resolve_text_data_root(launch_config: LaunchConfig) -> Path:
334
+ if hasattr(launch_config.data, "root_dir"):
335
+ return Path(launch_config.data.root_dir)
336
+ if hasattr(launch_config.data, "text"):
337
+ text_cfg = launch_config.data.text
338
+ root_dir = text_cfg.get("root_dir") if isinstance(text_cfg, dict) else getattr(text_cfg, "root_dir", None)
339
+ if root_dir:
340
+ return Path(root_dir)
341
+ raise AttributeError("run config has neither data.root_dir nor data.text.root_dir")
342
+
343
+
344
+ def load_prompt_initial_pose(
345
+ *,
346
+ args: argparse.Namespace,
347
+ data_root: Path,
348
+ metadata_csv: Path | None,
349
+ use_delta_actions: bool,
350
+ root_delta_actions: bool,
351
+ textop_actions: bool,
352
+ ) -> tuple[np.ndarray, str]:
353
+ if args.init_pose_npy:
354
+ # <modify> 允许prompt模式显式指定36D初始姿态,避免delta模型从zero起步
355
+ init_path = Path(args.init_pose_npy).expanduser()
356
+ if not init_path.is_absolute():
357
+ cwd_path = Path.cwd() / init_path
358
+ data_path = data_root / init_path
359
+ init_path = cwd_path if cwd_path.exists() else data_path
360
+ return load_pose_from_npy(init_path), str(init_path)
361
+
362
+ if args.init_motion_dir:
363
+ # <modify> 允许prompt模式显式指定SEED动作目录,用该动作第一帧作为初始姿态
364
+ motion_dir = resolve_motion_dir(data_root, args.init_motion_dir)
365
+ return load_pose_from_motion_dir(motion_dir), str(motion_dir)
366
+
367
+ if args.restore_from_data:
368
+ # <modify> 默认从metadata的start-index动作取真实首帧,不再静默使用zero
369
+ if metadata_csv is None:
370
+ raise ValueError(
371
+ "Prompt inference needs an initial pose. Use --init-motion-dir or --init-pose-npy "
372
+ "when the run config has no metadata_csv."
373
+ )
374
+ prompt_metadata = pd.read_csv(metadata_csv)
375
+ if len(prompt_metadata) == 0:
376
+ raise ValueError(f"Empty metadata csv: {metadata_csv}")
377
+ ref_index = min(max(0, int(args.start_index)), len(prompt_metadata) - 1)
378
+ ref_row = prompt_metadata.iloc[ref_index]
379
+ motion_dir = map_motion_dir(data_root, str(ref_row["move_g1_path"]))
380
+ return load_pose_from_motion_dir(motion_dir), f"{motion_dir} (metadata index {ref_index})"
381
+
382
+ # <modify_jerk> hybrid root-delta prompt推理同样需要真实初始root。
383
+ if use_delta_actions or root_delta_actions or textop_actions:
384
+ raise ValueError(
385
+ "Delta/root-delta/TextOp prompt inference requires a real initial pose. "
386
+ "Use --init-motion-dir, --init-pose-npy, or keep --restore-from-data."
387
+ )
388
+
389
+ return np.zeros((36,), dtype=np.float32), "zero"
390
+
391
+
392
+ def rollout_motion(
393
+ *,
394
+ model: Psi0Model,
395
+ maxmin,
396
+ odim: int,
397
+ device: str,
398
+ instruction: str,
399
+ text_only: bool,
400
+ normalize_state: bool,
401
+ use_delta_actions: bool,
402
+ root_delta_actions: bool,
403
+ textop_actions: bool,
404
+ decode_textop_joint_from_delta: bool,
405
+ num_past_frames: int,
406
+ use_future_progress_embedding: bool,
407
+ enable_rtc: bool,
408
+ action_exec_horizon: int,
409
+ rtc_max_delay: int,
410
+ num_inference_steps: int,
411
+ target_frames: int,
412
+ init_abs_state: np.ndarray,
413
+ ) -> tuple[np.ndarray, np.ndarray, int]:
414
+ horizon = int(model.action_horizon)
415
+ exec_horizon = int(action_exec_horizon) if enable_rtc else horizon
416
+ if exec_horizon <= 0 or exec_horizon > horizon:
417
+ raise ValueError(f"Invalid action_exec_horizon={exec_horizon}, action_horizon={horizon}")
418
+ rtc_delay = horizon - exec_horizon
419
+ if enable_rtc and rtc_delay > 0 and rtc_delay >= int(rtc_max_delay):
420
+ raise ValueError(
421
+ f"RTC requires action_horizon - action_exec_horizon < max_delay, "
422
+ f"got {horizon} - {exec_horizon} = {rtc_delay}, max_delay={rtc_max_delay}"
423
+ )
424
+ if target_frames <= 0:
425
+ target_frames = exec_horizon
426
+ # <modify> RTC推理按exec_horizon滚动;普通推理仍按完整action_horizon滚动
427
+ # 原逻辑:num_chunks = int(math.ceil(float(target_frames) / float(horizon)))
428
+ num_chunks = int(math.ceil(float(target_frames) / float(exec_horizon)))
429
+
430
+ current_abs = np.asarray(init_abs_state, dtype=np.float32).copy()
431
+ # <modify_states> 推理时维护和训练一致的多帧state历史,首个chunk用初始姿态重复填充。
432
+ state_horizon = int(num_past_frames) + 1
433
+ state_history: List[np.ndarray] = [current_abs.copy() for _ in range(state_horizon)]
434
+ all_abs: List[np.ndarray] = []
435
+ all_delta: List[np.ndarray] = []
436
+ prev_actions_norm: np.ndarray | None = None
437
+ remaining_frames = int(target_frames)
438
+
439
+ for _ in range(num_chunks):
440
+ generated_frames = int(target_frames - remaining_frames)
441
+ # <modify_states> 使用最近To帧state作为输入;保持shape(B, To, Ds)。
442
+ # 原逻辑:
443
+ # states = current_abs.reshape(1, odim).astype(np.float32)
444
+ states = build_model_states_from_abs(
445
+ state_history[-state_horizon:],
446
+ textop_actions=textop_actions,
447
+ odim=odim,
448
+ )
449
+ if normalize_state:
450
+ states = maxmin.normalize_state_func(states)
451
+ states_t = torch.from_numpy(states).unsqueeze(0).to(device)
452
+ future_progress_t = None
453
+ if use_future_progress_embedding:
454
+ _, future_progress = build_progress_conditions(
455
+ frame_id=generated_frames,
456
+ total_frames=target_frames,
457
+ action_horizon=horizon,
458
+ )
459
+ future_progress_t = torch.from_numpy(future_progress[None]).to(device)
460
+
461
+ with torch.inference_mode():
462
+ if enable_rtc and prev_actions_norm is not None and rtc_delay > 0:
463
+ # <modify> RTC推理把上一chunk尚未执行的尾部动作作为下一chunk的prefix条件
464
+ prev_rtc = np.concatenate(
465
+ [
466
+ prev_actions_norm[exec_horizon:, :],
467
+ np.zeros((exec_horizon, prev_actions_norm.shape[1]), dtype=np.float32),
468
+ ],
469
+ axis=0,
470
+ )
471
+ prev_actions_t = torch.from_numpy(prev_rtc[None]).to(device)
472
+ pred_norm = model.predict_action_with_training_rtc_flow(
473
+ observations=None if text_only else [[]],
474
+ states=states_t,
475
+ instructions=[instruction],
476
+ num_inference_steps=num_inference_steps,
477
+ traj2ds=None,
478
+ prev_actions=prev_actions_t,
479
+ inference_delay=rtc_delay,
480
+ max_delay=int(rtc_max_delay),
481
+ future_progress=future_progress_t,
482
+ )[0].float().cpu().numpy()
483
+ else:
484
+ pred_norm = model.predict_action(
485
+ observations=None if text_only else [[]],
486
+ states=states_t,
487
+ instructions=[instruction],
488
+ num_inference_steps=num_inference_steps,
489
+ traj2ds=None,
490
+ future_progress=future_progress_t,
491
+ )[0].float().cpu().numpy()
492
+
493
+ # <modify> 训练侧ActionStateTransform会把normalized action裁剪到[-1,1],推理反归一化前也要保持同一范围。
494
+ # 原逻辑:chunk_pred = np.asarray(maxmin.denormalize(pred_norm), dtype=np.float32)
495
+ pred_norm_clipped = np.clip(np.asarray(pred_norm, dtype=np.float32), -1.0, 1.0)
496
+ chunk_pred = np.asarray(maxmin.denormalize(pred_norm_clipped), dtype=np.float32)
497
+
498
+ # <modify_textop_repr> TextOp训练目标先恢复到36D绝对姿态;root旋转delta在helper里按相对旋转6D解码。
499
+ if textop_actions:
500
+ if decode_textop_joint_from_delta:
501
+ chunk_abs = recover_seed_abs_from_textop_action_with_joint_delta(chunk_pred, current_abs)
502
+ else:
503
+ chunk_abs = recover_seed_abs_from_textop_action(chunk_pred, current_abs)
504
+ prev_abs = np.concatenate([current_abs[None, :], chunk_abs[:-1]], axis=0)
505
+ joint_delta = chunk_abs[:, :29] - prev_abs[:, :29]
506
+ root_pos_delta = chunk_abs[:, 29:32] - prev_abs[:, 29:32]
507
+ root_quat_delta = quat_mul_wxyz(quat_inv_wxyz(prev_abs[:, 32:36]), chunk_abs[:, 32:36])
508
+ chunk_delta = np.concatenate([joint_delta, root_pos_delta, root_quat_delta], axis=1)
509
+ elif use_delta_actions:
510
+ # <modify> delta action需要从current_abs累加成绝对姿态,支持多chunk滚动
511
+ # 原逻辑:把chunk_pred直接当绝对姿态,和delta训练目标不一致
512
+ chunk_delta = chunk_pred
513
+ chunk_abs = np.cumsum(chunk_delta, axis=0) + current_abs[None, :]
514
+ elif root_delta_actions:
515
+ # <modify_jerk> hybrid action:关节直接用绝对预测,root 7维从当前姿态逐帧累加。
516
+ chunk_abs = chunk_pred.copy()
517
+ root_abs = current_abs[29:36].astype(np.float32).copy()
518
+ root_abs_seq = []
519
+ for root_delta in chunk_pred[:, 29:36]:
520
+ root_abs = root_abs + root_delta
521
+ quat_norm = np.linalg.norm(root_abs[3:7])
522
+ if quat_norm > 1e-8:
523
+ root_abs[3:7] = root_abs[3:7] / quat_norm
524
+ root_abs_seq.append(root_abs.copy())
525
+ chunk_abs[:, 29:36] = np.stack(root_abs_seq, axis=0)
526
+
527
+ first_joint_delta = chunk_abs[0, :29] - current_abs[:29]
528
+ if chunk_abs.shape[0] > 1:
529
+ rest_joint_delta = chunk_abs[1:, :29] - chunk_abs[:-1, :29]
530
+ joint_delta = np.concatenate([first_joint_delta[None, :], rest_joint_delta], axis=0)
531
+ else:
532
+ joint_delta = first_joint_delta[None, :]
533
+ chunk_delta = np.concatenate([joint_delta, chunk_pred[:, 29:36]], axis=1)
534
+ else:
535
+ chunk_abs = chunk_pred
536
+ first_delta = chunk_abs[0] - current_abs
537
+ if chunk_abs.shape[0] > 1:
538
+ rest_delta = chunk_abs[1:] - chunk_abs[:-1]
539
+ chunk_delta = np.concatenate([first_delta[None, :], rest_delta], axis=0)
540
+ else:
541
+ chunk_delta = first_delta[None, :]
542
+
543
+ chunk_abs = normalize_quaternion_rows(chunk_abs.astype(np.float32))
544
+ # <modify> TextOp且joint_abs未监督时,RTC prefix不能继续使用模型预测的joint_abs通道。
545
+ # 原逻辑:prev_actions_norm = np.asarray(pred_norm, dtype=np.float32).copy()
546
+ # 这里用joint_delta还原后的关节绝对值回填0:29,再重新normalize,避免坏joint_abs污染下一chunk。
547
+ if textop_actions and decode_textop_joint_from_delta:
548
+ chunk_pred_for_rtc = chunk_pred.copy()
549
+ chunk_pred_for_rtc[:, :29] = chunk_abs[:, :29]
550
+ # <modify> 这里不能直接调用maxmin.normalize,因为当前配置normalize_state=True时会要求data里有states。
551
+ action_min = np.asarray(maxmin.action_min, dtype=np.float32)
552
+ action_max = np.asarray(maxmin.action_max, dtype=np.float32)
553
+ ill_mask = np.abs(action_max - action_min) < 1e-4 * (np.abs(action_max) + np.abs(action_min) + 1e-8)
554
+ action_max = action_max.copy()
555
+ action_max[ill_mask] = 1.0
556
+ chunk_pred_for_rtc_norm = np.where(
557
+ ill_mask,
558
+ chunk_pred_for_rtc,
559
+ (chunk_pred_for_rtc - action_min) / (action_max - action_min) * 2 - 1,
560
+ )
561
+ if bool(getattr(maxmin, "use_norm_mask", False)):
562
+ action_norm_masks = np.asarray(maxmin.action_norm_masks, dtype=bool)
563
+ chunk_pred_for_rtc_norm = np.where(action_norm_masks, chunk_pred_for_rtc_norm, chunk_pred_for_rtc)
564
+ prev_actions_norm = np.clip(chunk_pred_for_rtc_norm, -1, 1).astype(np.float32).copy()
565
+ else:
566
+ # <modify> RTC prefix也使用裁剪后的normalized action,避免把越界预测作为clean prefix条件。
567
+ # 原逻辑:prev_actions_norm = np.asarray(pred_norm, dtype=np.float32).copy()
568
+ prev_actions_norm = pred_norm_clipped.copy()
569
+ # <modify> RTC只导出本轮实际执行的前exec_horizon帧;普通模式exec_horizon等于horizon
570
+ take = min(exec_horizon, remaining_frames)
571
+ if take <= 0:
572
+ break
573
+ executed_abs = chunk_abs[:take]
574
+ executed_delta = chunk_delta[:take].astype(np.float32)
575
+ current_abs = executed_abs[-1].copy()
576
+ # <modify_states> 每执行一步都推进state历史,下一chunk看到的是真实rollout历史而不是单帧。
577
+ for frame_abs in executed_abs:
578
+ state_history.append(frame_abs.astype(np.float32).copy())
579
+ state_history = state_history[-state_horizon:]
580
+ all_abs.append(executed_abs)
581
+ all_delta.append(executed_delta)
582
+ remaining_frames -= take
583
+
584
+ pred_abs = np.concatenate(all_abs, axis=0)[:target_frames]
585
+ pred_delta = np.concatenate(all_delta, axis=0)[:target_frames]
586
+ pred_abs = normalize_quaternion_rows(pred_abs.astype(np.float32))
587
+ return pred_abs, pred_delta, num_chunks
588
+
589
+
590
+ def resolve_ckpt_step(run_dir: Path, ckpt_step: str) -> int | str:
591
+ if ckpt_step == "best":
592
+ # <modify> 支持--ckpt-step best,直接读取checkpoints/best_ckpt目录。
593
+ best_ckpt = run_dir / "checkpoints" / "best_ckpt"
594
+ if not best_ckpt.exists():
595
+ raise RuntimeError(f"No best_ckpt found under {run_dir / 'checkpoints'}")
596
+ return "best_ckpt"
597
+
598
+ if ckpt_step != "latest":
599
+ return int(ckpt_step)
600
+
601
+ ckpt_root = run_dir / "checkpoints"
602
+ candidates = []
603
+ for p in ckpt_root.iterdir():
604
+ if p.is_dir() and p.name.startswith("ckpt_"):
605
+ try:
606
+ candidates.append(int(p.name.split("_")[-1]))
607
+ except ValueError:
608
+ continue
609
+ if not candidates:
610
+ raise RuntimeError(f"No ckpt_* found under {ckpt_root}")
611
+ return max(candidates)
612
+
613
+
614
+ def parse_args() -> argparse.Namespace:
615
+ parser = argparse.ArgumentParser(description="Seed text->motion offline inference exporter")
616
+ parser.add_argument("--run-dir", type=str, required=True)
617
+ parser.add_argument("--ckpt-step", type=str, default="latest", help="integer step, 'latest', or 'best'")
618
+ parser.add_argument("--output-dir", type=str, required=True)
619
+ parser.add_argument("--num-samples", type=int, default=1)
620
+ parser.add_argument("--start-index", type=int, default=0)
621
+ parser.add_argument("--num-inference-steps", type=int, default=10)
622
+ parser.add_argument("--device", type=str, default="cuda:0")
623
+ parser.add_argument("--metadata-csv", type=str, default="")
624
+ parser.add_argument("--prompt", type=str, default="", help="Direct text prompt for manual inference")
625
+ parser.add_argument("--prompt-motion-name", type=str, default="manual_prompt")
626
+ parser.add_argument("--init-motion-dir", type=str, default="", help="SEED csv_sonic motion dir used for prompt initial pose")
627
+ parser.add_argument("--init-pose-npy", type=str, default="", help="36D pose npy, or Tx36 motion npy, used for prompt initial pose")
628
+ parser.add_argument("--num-output-frames", type=int, default=0, help="0 means auto length")
629
+ parser.add_argument("--seed", type=int, default=42)
630
+ parser.add_argument("--restore-from-data", action="store_true", default=True)
631
+ parser.add_argument("--no-restore-from-data", dest="restore_from_data", action="store_false")
632
+ parser.add_argument("--export-delta", action="store_true", default=False)
633
+ parser.add_argument("--disable-rtc", action="store_true", default=False)
634
+ parser.add_argument(
635
+ "--force-text-only",
636
+ action="store_true",
637
+ default=False,
638
+ help="Use the same checkpoint but force text-only VLM inputs, even if run_config is audio-only.",
639
+ )
640
+ return parser.parse_args()
641
+
642
+
643
+ def main() -> None:
644
+ args = parse_args()
645
+
646
+ run_dir = Path(args.run_dir)
647
+ output_dir = Path(args.output_dir)
648
+ output_dir.mkdir(parents=True, exist_ok=True)
649
+
650
+ cfg_template: LaunchConfig = parse_args_to_tyro_config(run_dir / "argv.txt") # type: ignore
651
+ launch_config = cfg_template.model_validate_json((run_dir / "run_config.json").read_text())
652
+
653
+ seed_everything(launch_config.seed or args.seed)
654
+
655
+ # 找到指定ckpt
656
+ ckpt_step = resolve_ckpt_step(run_dir, args.ckpt_step)
657
+ model = Psi0Model.from_pretrained(run_dir, ckpt_step, launch_config, device=args.device)
658
+ if args.force_text_only:
659
+ model.text_only = True
660
+ model.audio_only = False
661
+ model.video_only = False
662
+ model.to(args.device)
663
+ model.eval()
664
+
665
+ data_root = resolve_text_data_root(launch_config)
666
+ prompt_mode = bool(args.prompt.strip())
667
+ use_delta_actions = bool(getattr(launch_config.data.transform.repack, "use_delta_actions", False))
668
+ # <modify_jerk> run_config里记录hybrid action模式:关节绝对,root delta。
669
+ root_delta_actions = bool(getattr(launch_config.data.transform.repack, "root_delta_actions", False))
670
+ # TextOp模式下训练目标是39D action/state;推理端用同一个decoder恢复为36D绝对动作。
671
+ textop_actions = bool(getattr(launch_config.data.transform.repack, "textop_actions", False))
672
+ decode_textop_joint_from_delta = textop_actions and should_decode_textop_joint_from_delta(launch_config)
673
+ # <modify_states> 从run_config读取SEED多帧state配置;旧checkpoint默认退化为单帧state。
674
+ num_past_frames = int(getattr(launch_config.data.transform.repack, "num_past_frames", 0))
675
+ # <modify> run_config里model.rtc=True时默认使用RTC滚动推理;可用--disable-rtc关闭
676
+ enable_rtc = bool(getattr(launch_config.model, "rtc", False)) and (not args.disable_rtc)
677
+ use_future_progress_embedding = bool(getattr(launch_config.model, "use_future_progress_embedding", False))
678
+ action_exec_horizon = int(getattr(launch_config.model, "action_exec_horizon", int(model.action_horizon)))
679
+ rtc_max_delay = int(getattr(launch_config.model, "max_delay", 0))
680
+ metadata_csv = resolve_metadata_csv(data_root, launch_config, args.metadata_csv)
681
+
682
+ metadata = None
683
+ if not prompt_mode:
684
+ metadata = pd.read_csv(metadata_csv)
685
+ start = max(0, args.start_index)
686
+ end = min(len(metadata), start + max(0, args.num_samples))
687
+ if start >= end:
688
+ raise ValueError(f"Empty sampling range: start={start}, end={end}, metadata_size={len(metadata)}")
689
+ sample_indices = list(range(start, end))
690
+ else:
691
+ sample_indices = [0]
692
+
693
+ text_only = bool(args.force_text_only) or bool(getattr(launch_config.data.transform.model, "text_only", False))
694
+ maxmin = launch_config.data.transform.field
695
+ odim = int(launch_config.model.odim)
696
+ action_dim = int(getattr(launch_config.model, "action_dim", getattr(model, "action_dim", SEED_TEXTOP_ACTION_DIM)))
697
+ if textop_actions and (action_dim != SEED_TEXTOP_ACTION_DIM or odim != SEED_TEXTOP_ACTION_DIM):
698
+ raise ValueError(
699
+ f"TextOp infer/train dim mismatch: action_dim={action_dim}, odim={odim}, expected={SEED_TEXTOP_ACTION_DIM}"
700
+ )
701
+
702
+ manifest = {
703
+ "run_dir": str(run_dir),
704
+ "ckpt_step": ckpt_step,
705
+ "metadata_csv": str(metadata_csv) if metadata_csv is not None else "",
706
+ "mode": "prompt" if prompt_mode else "metadata",
707
+ "text_only": text_only,
708
+ "action_dim": int(action_dim),
709
+ "state_dim": int(odim),
710
+ "use_delta_actions": use_delta_actions,
711
+ "root_delta_actions": root_delta_actions,
712
+ "textop_actions": textop_actions,
713
+ "decode_textop_joint_from_delta": decode_textop_joint_from_delta,
714
+ "num_past_frames": int(num_past_frames),
715
+ "use_future_progress_embedding": bool(use_future_progress_embedding),
716
+ "enable_rtc": enable_rtc,
717
+ "action_exec_horizon": int(action_exec_horizon),
718
+ "rtc_max_delay": int(rtc_max_delay),
719
+ "num_inference_steps": int(args.num_inference_steps),
720
+ "num_output_frames": int(args.num_output_frames),
721
+ "export_delta": bool(args.export_delta),
722
+ "samples": [],
723
+ }
724
+
725
+ for idx in sample_indices:
726
+ if prompt_mode:
727
+ raw_instruction = args.prompt.strip()
728
+ instruction = compose_seed_motion_prompt(raw_instruction)
729
+ source_motion_dir = Path("manual_prompt")
730
+ # <modify> prompt模式下delta推理不能再用zero初始姿态,默认从metadata的start-index动作取真实首帧
731
+ # 原逻辑:x0 = np.zeros((36,), dtype=np.float32)
732
+ x0, initial_pose_source = load_prompt_initial_pose(
733
+ args=args,
734
+ data_root=data_root,
735
+ metadata_csv=metadata_csv,
736
+ use_delta_actions=use_delta_actions,
737
+ root_delta_actions=root_delta_actions,
738
+ textop_actions=textop_actions,
739
+ )
740
+ auto_frames = max(int(model.action_horizon), 300)
741
+ target_frames = int(args.num_output_frames) if args.num_output_frames > 0 else auto_frames
742
+ motion_name = sanitize_motion_name(args.prompt_motion_name)
743
+ sample_dir = output_dir / "g1" / "csv_sonic" / "manual" / motion_name
744
+ else:
745
+ assert metadata is not None
746
+ row = metadata.iloc[idx]
747
+ raw_instruction = ""
748
+ instruction = compose_instruction(row, DEFAULT_TEXT_FIELDS)
749
+ source_motion_dir = map_motion_dir(data_root, str(row.get("move_g1_path", "")))
750
+ # <modify> delta推理加载初始姿态失败时直接报错,避免静默退回zero
751
+ x0 = load_initial_pose(
752
+ data_root,
753
+ row,
754
+ restore_from_data=args.restore_from_data,
755
+ strict=(use_delta_actions or root_delta_actions or textop_actions),
756
+ )
757
+ initial_pose_source = str(source_motion_dir) if args.restore_from_data else "zero"
758
+ source_frames = load_source_frame_count(source_motion_dir)
759
+ if args.num_output_frames > 0:
760
+ target_frames = int(args.num_output_frames)
761
+ elif source_frames is not None:
762
+ target_frames = source_frames
763
+ else:
764
+ target_frames = int(model.action_horizon)
765
+ # Output folder keeps the same relative hierarchy as csv_sonic
766
+ try:
767
+ rel_dir = source_motion_dir.relative_to(data_root)
768
+ except Exception:
769
+ rel_dir = Path(f"sample_{idx:04d}")
770
+ sample_dir = output_dir / rel_dir
771
+ motion_name = source_motion_dir.name if source_motion_dir.name else f"sample_{idx:04d}"
772
+
773
+ pred_abs, pred_delta, rollout_chunks = rollout_motion(
774
+ model=model,
775
+ maxmin=maxmin,
776
+ odim=odim,
777
+ device=args.device,
778
+ instruction=instruction,
779
+ text_only=text_only,
780
+ normalize_state=bool(getattr(maxmin, "normalize_state", False)),
781
+ use_delta_actions=use_delta_actions,
782
+ root_delta_actions=root_delta_actions,
783
+ textop_actions=textop_actions,
784
+ decode_textop_joint_from_delta=decode_textop_joint_from_delta,
785
+ num_past_frames=num_past_frames,
786
+ use_future_progress_embedding=use_future_progress_embedding,
787
+ enable_rtc=enable_rtc,
788
+ action_exec_horizon=action_exec_horizon,
789
+ rtc_max_delay=rtc_max_delay,
790
+ num_inference_steps=int(args.num_inference_steps),
791
+ target_frames=target_frames,
792
+ init_abs_state=x0,
793
+ )
794
+
795
+ sample_dir.mkdir(parents=True, exist_ok=True)
796
+
797
+ joint_pos = pred_abs[:, :29]
798
+ body_pos = pred_abs[:, 29:32]
799
+ body_quat = pred_abs[:, 32:36]
800
+
801
+ dt = 1.0 / float(TARGET_FPS)
802
+ joint_vel = finite_difference(joint_pos, dt)
803
+ body_lin_vel = finite_difference(body_pos, dt)
804
+ body_ang_vel = quaternion_angular_velocity(body_quat, dt)
805
+
806
+ # Main outputs: match csv_sonic naming/headers
807
+ write_csv(sample_dir / "joint_pos.csv", joint_pos, [f"joint_{i}" for i in range(29)])
808
+ write_csv(sample_dir / "joint_vel.csv", joint_vel, [f"joint_vel_{i}" for i in range(29)])
809
+ write_csv(sample_dir / "body_pos.csv", body_pos, ["body_0_x", "body_0_y", "body_0_z"])
810
+ write_csv(sample_dir / "body_quat.csv", body_quat, ["body_0_w", "body_0_x", "body_0_y", "body_0_z"])
811
+ write_csv(sample_dir / "body_lin_vel.csv", body_lin_vel, ["body_0_vel_x", "body_0_vel_y", "body_0_vel_z"])
812
+ write_csv(sample_dir / "body_ang_vel.csv", body_ang_vel, ["body_0_angvel_x", "body_0_angvel_y", "body_0_angvel_z"])
813
+
814
+ np.save(sample_dir / "motion.npy", pred_abs)
815
+
816
+ # Optional delta outputs
817
+ if args.export_delta:
818
+ write_csv(sample_dir / "joint_pos_delta.csv", pred_delta[:, :29], [f"joint_{i}" for i in range(29)])
819
+ write_csv(sample_dir / "body_pos_delta.csv", pred_delta[:, 29:32], ["body_0_x", "body_0_y", "body_0_z"])
820
+ write_csv(sample_dir / "body_quat_delta.csv", pred_delta[:, 32:36], ["body_0_w", "body_0_x", "body_0_y", "body_0_z"])
821
+ np.save(sample_dir / "motion_delta.npy", pred_delta)
822
+
823
+ write_metadata(sample_dir / "metadata.txt", motion_name, int(pred_abs.shape[0]))
824
+ write_info(
825
+ sample_dir / "info.txt",
826
+ source_path=str(source_motion_dir),
827
+ initial_pose_source=initial_pose_source,
828
+ frames=int(pred_abs.shape[0]),
829
+ use_delta_actions=use_delta_actions,
830
+ root_delta_actions=root_delta_actions,
831
+ textop_actions=textop_actions,
832
+ num_past_frames=int(num_past_frames),
833
+ enable_rtc=enable_rtc,
834
+ action_exec_horizon=int(action_exec_horizon),
835
+ num_inference_steps=int(args.num_inference_steps),
836
+ )
837
+
838
+ # Keep source references if existing
839
+ if not prompt_mode:
840
+ src_info = source_motion_dir / "info.txt"
841
+ src_meta = source_motion_dir / "metadata.txt"
842
+ if src_info.exists():
843
+ shutil.copy2(src_info, sample_dir / "info_source.txt")
844
+ if src_meta.exists():
845
+ shutil.copy2(src_meta, sample_dir / "metadata_source.txt")
846
+
847
+ sample_meta = {
848
+ "sample_index": int(idx),
849
+ "instruction": instruction,
850
+ "raw_instruction": raw_instruction,
851
+ "move_g1_path": "" if prompt_mode else str(metadata.iloc[idx].get("move_g1_path", "")),
852
+ "source_motion_dir": str(source_motion_dir),
853
+ "initial_pose_source": initial_pose_source,
854
+ "num_past_frames": int(num_past_frames),
855
+ "root_delta_actions": root_delta_actions,
856
+ "textop_actions": textop_actions,
857
+ "decode_textop_joint_from_delta": decode_textop_joint_from_delta,
858
+ "enable_rtc": enable_rtc,
859
+ "action_exec_horizon": int(action_exec_horizon),
860
+ "output_dir": str(sample_dir),
861
+ "num_output_frames": int(pred_abs.shape[0]),
862
+ "rollout_chunks": int(rollout_chunks),
863
+ }
864
+ (sample_dir / "infer_meta.json").write_text(json.dumps(sample_meta, ensure_ascii=False, indent=2))
865
+ manifest["samples"].append(sample_meta)
866
+ print(f"[OK] sample={idx} -> {sample_dir}")
867
+
868
+ (output_dir / "manifest.json").write_text(json.dumps(manifest, ensure_ascii=False, indent=2))
869
+ print(f"[DONE] wrote manifest: {output_dir / 'manifest.json'}")
870
+
871
+
872
+ if __name__ == "__main__":
873
+ main()
scripts/infer/seed_video_to_motion.py ADDED
@@ -0,0 +1,548 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/usr/bin/env python3
2
+ from __future__ import annotations
3
+
4
+ import argparse
5
+ import json
6
+ import math
7
+ import shutil
8
+ from pathlib import Path
9
+ from typing import List
10
+
11
+ import numpy as np
12
+ import torch
13
+
14
+ from psi.config.config import LaunchConfig
15
+ from psi.data.seed.motion_repr import quat_inv_wxyz, quat_mul_wxyz, recover_seed_abs_from_textop_action
16
+ from psi.data.seed.progress import build_progress_conditions
17
+ from psi.data.seed.seed_video_motion_dataset import (
18
+ _probe_video,
19
+ load_video_array,
20
+ load_video_clip_array,
21
+ load_video_embed_cache,
22
+ video_embed_cache_path,
23
+ )
24
+ from psi.models.motion_distill import MotionDistillPsi0Model
25
+ from psi.models.psi0 import Psi0Model
26
+ from psi.utils import parse_args_to_tyro_config, seed_everything
27
+
28
+ from seed_audio_to_motion import resolve_motion_dir, resolve_video_path
29
+ from seed_text_to_motion import (
30
+ TARGET_FPS,
31
+ build_model_states_from_abs,
32
+ finite_difference,
33
+ load_pose_from_motion_dir,
34
+ load_source_frame_count,
35
+ normalize_quaternion_rows,
36
+ quaternion_angular_velocity,
37
+ recover_seed_abs_from_textop_action_with_joint_delta,
38
+ resolve_ckpt_step,
39
+ sanitize_motion_name,
40
+ should_decode_textop_joint_from_delta,
41
+ write_csv,
42
+ write_metadata,
43
+ )
44
+
45
+
46
+ def cfg_get(obj, key: str, default=None):
47
+ if isinstance(obj, dict):
48
+ return obj.get(key, default)
49
+ return getattr(obj, key, default)
50
+
51
+
52
+ def video_data_cfg(launch_config: LaunchConfig, config_key: str = "video"):
53
+ return cfg_get(launch_config.data, config_key, cfg_get(launch_config.data, "video", launch_config.data))
54
+
55
+
56
+ def rollout_video_motion(
57
+ *,
58
+ model: Psi0Model,
59
+ maxmin,
60
+ odim: int,
61
+ device: str,
62
+ video: np.ndarray | None,
63
+ video_embed: dict | None,
64
+ video_path: Path,
65
+ instruction: str,
66
+ video_size_pixels: int,
67
+ video_sample_fps: float,
68
+ video_decoder: str,
69
+ video_pixel_cache_dtype: str,
70
+ video_clip_mode: bool,
71
+ video_clip_motion_fps: float,
72
+ normalize_state: bool,
73
+ textop_actions: bool,
74
+ decode_textop_joint_from_delta: bool,
75
+ num_past_frames: int,
76
+ use_future_progress_embedding: bool,
77
+ enable_rtc: bool,
78
+ action_exec_horizon: int,
79
+ rtc_max_delay: int,
80
+ num_inference_steps: int,
81
+ target_frames: int,
82
+ init_abs_state: np.ndarray,
83
+ ) -> tuple[np.ndarray, np.ndarray, int]:
84
+ horizon = int(model.action_horizon)
85
+ exec_horizon = int(action_exec_horizon) if enable_rtc else horizon
86
+ if exec_horizon <= 0 or exec_horizon > horizon:
87
+ raise ValueError(f"Invalid action_exec_horizon={exec_horizon}, action_horizon={horizon}")
88
+ rtc_delay = horizon - exec_horizon
89
+ if enable_rtc and rtc_delay > 0 and rtc_delay >= int(rtc_max_delay):
90
+ raise ValueError(
91
+ f"RTC requires action_horizon - action_exec_horizon < max_delay, "
92
+ f"got {horizon} - {exec_horizon} = {rtc_delay}, max_delay={rtc_max_delay}"
93
+ )
94
+ if not textop_actions:
95
+ raise ValueError("This video inference path is for the 39D TextOp action checkpoint.")
96
+ if target_frames <= 0:
97
+ target_frames = exec_horizon
98
+
99
+ num_chunks = int(math.ceil(float(target_frames) / float(exec_horizon)))
100
+ current_abs = np.asarray(init_abs_state, dtype=np.float32).copy()
101
+ state_horizon = int(num_past_frames) + 1
102
+ state_history: List[np.ndarray] = [current_abs.copy() for _ in range(state_horizon)]
103
+ all_abs: List[np.ndarray] = []
104
+ all_delta: List[np.ndarray] = []
105
+ prev_actions_norm: np.ndarray | None = None
106
+ remaining_frames = int(target_frames)
107
+
108
+ for _ in range(num_chunks):
109
+ if video_clip_mode:
110
+ chunk_index = len(all_abs)
111
+ video_clip_start_sec = float(chunk_index * exec_horizon) / float(video_clip_motion_fps)
112
+ video_clip_duration_sec = float(horizon) / float(video_clip_motion_fps)
113
+ chunk_video, _, _ = load_video_clip_array(
114
+ video_path,
115
+ start_sec=video_clip_start_sec,
116
+ duration_sec=video_clip_duration_sec,
117
+ video_sample_fps=video_sample_fps,
118
+ video_size_pixels=video_size_pixels,
119
+ video_decoder=video_decoder,
120
+ )
121
+ chunk_video_embed = None
122
+ else:
123
+ chunk_video = video
124
+ chunk_video_embed = video_embed
125
+
126
+ frame_cursor = target_frames - remaining_frames
127
+ states = build_model_states_from_abs(
128
+ state_history[-state_horizon:],
129
+ textop_actions=textop_actions,
130
+ odim=odim,
131
+ )
132
+ if normalize_state:
133
+ states = maxmin.normalize_state_func(states)
134
+ states_t = torch.from_numpy(states).unsqueeze(0).to(device)
135
+ future_progress_t = None
136
+ if use_future_progress_embedding:
137
+ if video_clip_mode:
138
+ # Training video_clip samples are cropped to one local horizon:
139
+ # state frame 0 plus action frames 1..horizon. Keep progress local
140
+ # at inference so action-token positions match the training distribution.
141
+ progress_frame_id = 0
142
+ progress_total_frames = horizon + 1
143
+ else:
144
+ progress_frame_id = frame_cursor
145
+ progress_total_frames = target_frames
146
+ _, future_progress = build_progress_conditions(
147
+ frame_id=progress_frame_id,
148
+ total_frames=progress_total_frames,
149
+ action_horizon=horizon,
150
+ )
151
+ future_progress_t = torch.from_numpy(future_progress[None]).to(device)
152
+
153
+ with torch.inference_mode():
154
+ if enable_rtc and prev_actions_norm is not None and rtc_delay > 0:
155
+ prev_rtc = np.concatenate(
156
+ [
157
+ prev_actions_norm[exec_horizon:, :],
158
+ np.zeros((exec_horizon, prev_actions_norm.shape[1]), dtype=np.float32),
159
+ ],
160
+ axis=0,
161
+ )
162
+ prev_actions_t = torch.from_numpy(prev_rtc[None]).to(device)
163
+ pred_norm = model.predict_action_with_training_rtc_flow(
164
+ observations=None,
165
+ states=states_t,
166
+ instructions=[instruction],
167
+ num_inference_steps=num_inference_steps,
168
+ traj2ds=None,
169
+ prev_actions=prev_actions_t,
170
+ inference_delay=rtc_delay,
171
+ max_delay=int(rtc_max_delay),
172
+ videos=[chunk_video] if chunk_video_embed is None else None,
173
+ video_embeds=[chunk_video_embed] if chunk_video_embed is not None else None,
174
+ video_paths=[str(video_path)],
175
+ video_size_pixels=int(video_size_pixels),
176
+ video_sample_fps=float(video_sample_fps),
177
+ video_pixel_cache_dtype=video_pixel_cache_dtype,
178
+ future_progress=future_progress_t,
179
+ )[0].float().cpu().numpy()
180
+ else:
181
+ pred_norm = model.predict_action(
182
+ observations=None,
183
+ states=states_t,
184
+ instructions=[instruction],
185
+ num_inference_steps=num_inference_steps,
186
+ traj2ds=None,
187
+ videos=[chunk_video] if chunk_video_embed is None else None,
188
+ video_embeds=[chunk_video_embed] if chunk_video_embed is not None else None,
189
+ video_paths=[str(video_path)],
190
+ video_size_pixels=int(video_size_pixels),
191
+ video_sample_fps=float(video_sample_fps),
192
+ video_pixel_cache_dtype=video_pixel_cache_dtype,
193
+ future_progress=future_progress_t,
194
+ )[0].float().cpu().numpy()
195
+
196
+ pred_norm_clipped = np.clip(np.asarray(pred_norm, dtype=np.float32), -1.0, 1.0)
197
+ chunk_pred = np.asarray(maxmin.denormalize(pred_norm_clipped), dtype=np.float32)
198
+
199
+ if decode_textop_joint_from_delta:
200
+ chunk_abs = recover_seed_abs_from_textop_action_with_joint_delta(chunk_pred, current_abs)
201
+ else:
202
+ chunk_abs = recover_seed_abs_from_textop_action(chunk_pred, current_abs)
203
+
204
+ prev_abs = np.concatenate([current_abs[None, :], chunk_abs[:-1]], axis=0)
205
+ joint_delta = chunk_abs[:, :29] - prev_abs[:, :29]
206
+ root_pos_delta = chunk_abs[:, 29:32] - prev_abs[:, 29:32]
207
+ root_quat_delta = quat_mul_wxyz(quat_inv_wxyz(prev_abs[:, 32:36]), chunk_abs[:, 32:36])
208
+ chunk_delta = np.concatenate([joint_delta, root_pos_delta, root_quat_delta], axis=1)
209
+
210
+ chunk_abs = normalize_quaternion_rows(chunk_abs.astype(np.float32))
211
+ if decode_textop_joint_from_delta:
212
+ chunk_pred_for_rtc = chunk_pred.copy()
213
+ chunk_pred_for_rtc[:, :29] = chunk_abs[:, :29]
214
+ from seed_audio_to_motion import normalize_action_for_rtc
215
+ prev_actions_norm = normalize_action_for_rtc(maxmin, chunk_pred_for_rtc)
216
+ else:
217
+ prev_actions_norm = pred_norm_clipped.copy()
218
+
219
+ take = min(exec_horizon, remaining_frames)
220
+ if take <= 0:
221
+ break
222
+ executed_abs = chunk_abs[:take]
223
+ executed_delta = chunk_delta[:take].astype(np.float32)
224
+ current_abs = executed_abs[-1].copy()
225
+ for frame_abs in executed_abs:
226
+ state_history.append(frame_abs.astype(np.float32).copy())
227
+ state_history = state_history[-state_horizon:]
228
+ all_abs.append(executed_abs)
229
+ all_delta.append(executed_delta)
230
+ remaining_frames -= take
231
+
232
+ pred_abs = np.concatenate(all_abs, axis=0)[:target_frames]
233
+ pred_delta = np.concatenate(all_delta, axis=0)[:target_frames]
234
+ pred_abs = normalize_quaternion_rows(pred_abs.astype(np.float32))
235
+ return pred_abs, pred_delta, num_chunks
236
+
237
+
238
+ def write_video_info(
239
+ path: Path,
240
+ *,
241
+ video_decoder: str,
242
+ source_motion_dir: Path,
243
+ video_path: Path,
244
+ instruction: str,
245
+ initial_pose_source: str,
246
+ frames: int,
247
+ textop_actions: bool,
248
+ num_past_frames: int,
249
+ enable_rtc: bool,
250
+ action_exec_horizon: int,
251
+ num_inference_steps: int,
252
+ video_sample_fps: float,
253
+ video_size_pixels: int,
254
+ video_pixel_cache_dtype: str,
255
+ video_embed_path: str,
256
+ source_video_fps: float,
257
+ video_duration_sec: float,
258
+ full_video: bool,
259
+ video_config_key: str,
260
+ video_clip_motion_fps: float,
261
+ ) -> None:
262
+ lines = [
263
+ "Motion Information: seed_video_to_motion_inference",
264
+ "===================================================",
265
+ "",
266
+ f"video_decoder: {video_decoder}",
267
+ f"video_config_key: {video_config_key}",
268
+ f"source_motion_dir: {source_motion_dir}",
269
+ f"video_path: {video_path}",
270
+ f"instruction: {instruction}",
271
+ f"initial_pose_source: {initial_pose_source}",
272
+ f"target_fps: {TARGET_FPS}",
273
+ f"num_inference_steps: {num_inference_steps}",
274
+ f"trained_with_textop_actions: {textop_actions}",
275
+ f"num_past_frames: {num_past_frames}",
276
+ f"rtc_enabled: {enable_rtc}",
277
+ f"action_exec_horizon: {action_exec_horizon}",
278
+ f"video_sample_fps: {video_sample_fps}",
279
+ f"video_size_pixels: {video_size_pixels}",
280
+ f"video_pixel_cache_dtype: {video_pixel_cache_dtype}",
281
+ f"video_embed_path: {video_embed_path}",
282
+ f"source_video_fps: {source_video_fps}",
283
+ f"video_duration_sec: {video_duration_sec}",
284
+ f"full_video: {full_video}",
285
+ f"video_clip_motion_fps: {video_clip_motion_fps}",
286
+ "use_audio_in_video: False",
287
+ "output_root_quaternion: wxyz",
288
+ "output_joint_angles: 29D",
289
+ "output_root_position: xyz",
290
+ "",
291
+ f"frames: {frames}",
292
+ "",
293
+ ]
294
+ path.write_text("\n".join(lines), encoding="utf-8")
295
+
296
+
297
+ def parse_args() -> argparse.Namespace:
298
+ parser = argparse.ArgumentParser(description="Seed video->motion offline inference exporter")
299
+ parser.add_argument("--run-dir", type=str, required=True)
300
+ parser.add_argument("--ckpt-step", type=str, default="latest", help="integer step, 'latest', or 'best'")
301
+ parser.add_argument("--output-dir", type=str, required=True)
302
+ parser.add_argument("--motion-dir", type=str, required=True)
303
+ parser.add_argument("--video-path", type=str, default="")
304
+ parser.add_argument("--num-output-frames", type=int, default=0, help="0 means source motion length")
305
+ parser.add_argument("--num-inference-steps", type=int, default=10)
306
+ parser.add_argument("--device", type=str, default="cuda:0")
307
+ parser.add_argument("--seed", type=int, default=42)
308
+ parser.add_argument("--prompt", type=str, default="", help="Optional text instruction paired with the video")
309
+ parser.add_argument("--export-delta", action="store_true", default=False)
310
+ parser.add_argument("--disable-rtc", action="store_true", default=False)
311
+ parser.add_argument("--video-config-key", type=str, default="video", help="data config key: video or video_clip")
312
+ parser.add_argument("--video-clip", action="store_true", default=False, help="Use one aligned video clip per rollout chunk")
313
+ parser.add_argument(
314
+ "--video-pixel-cache-dtype",
315
+ choices=("auto", "none", "fp16", "bf16"),
316
+ default="auto",
317
+ help="Match cached Qwen pixel-value quantization; auto uses fp16 for cached video_clip runs.",
318
+ )
319
+ parser.add_argument(
320
+ "--video-clip-motion-fps",
321
+ type=float,
322
+ default=0.0,
323
+ help="Override motion fps used to align rollout chunks to video clips; <=0 infers it from source motion/video.",
324
+ )
325
+ return parser.parse_args()
326
+
327
+
328
+ def main() -> None:
329
+ args = parse_args()
330
+
331
+ run_dir = Path(args.run_dir)
332
+ output_dir = Path(args.output_dir)
333
+ output_dir.mkdir(parents=True, exist_ok=True)
334
+
335
+ cfg_template: LaunchConfig = parse_args_to_tyro_config(run_dir / "argv.txt") # type: ignore
336
+ launch_config = cfg_template.model_validate_json((run_dir / "run_config.json").read_text())
337
+ seed_everything(launch_config.seed or args.seed)
338
+
339
+ ckpt_step = resolve_ckpt_step(run_dir, args.ckpt_step)
340
+ model_cls = (
341
+ MotionDistillPsi0Model
342
+ if hasattr(launch_config.model, "motion_distill_stage")
343
+ else Psi0Model
344
+ )
345
+ model = model_cls.from_pretrained(run_dir, ckpt_step, launch_config, device=args.device)
346
+ model.text_only = False
347
+ model.audio_only = False
348
+ model.video_only = True
349
+ model.to(args.device)
350
+ model.eval()
351
+
352
+ video_cfg = video_data_cfg(launch_config, args.video_config_key)
353
+ data_root = Path(cfg_get(video_cfg, "root_dir"))
354
+ video_root = Path(cfg_get(video_cfg, "video_dir", ""))
355
+ motion_dir = resolve_motion_dir(data_root, args.motion_dir)
356
+ video_path = resolve_video_path(video_root, motion_dir, args.video_path)
357
+ if not motion_dir.exists():
358
+ raise FileNotFoundError(f"motion_dir does not exist: {motion_dir}")
359
+ if not video_path.exists():
360
+ raise FileNotFoundError(f"video_path does not exist: {video_path}")
361
+
362
+ maxmin = launch_config.data.transform.field
363
+ odim = int(launch_config.model.odim)
364
+ textop_actions = bool(getattr(launch_config.data.transform.repack, "textop_actions", False))
365
+ decode_textop_joint_from_delta = textop_actions and should_decode_textop_joint_from_delta(launch_config)
366
+ num_past_frames = int(getattr(launch_config.data.transform.repack, "num_past_frames", 0))
367
+ use_future_progress_embedding = bool(getattr(launch_config.model, "use_future_progress_embedding", False))
368
+ enable_rtc = bool(getattr(launch_config.model, "rtc", False)) and (not args.disable_rtc)
369
+ action_exec_horizon = int(getattr(launch_config.model, "action_exec_horizon", int(model.action_horizon)))
370
+ rtc_max_delay = int(getattr(launch_config.model, "max_delay", 0))
371
+ video_sample_fps = float(cfg_get(video_cfg, "video_sample_fps", 1.0))
372
+ video_size_pixels = int(cfg_get(video_cfg, "video_size_pixels", 50176))
373
+ video_decoder = str(cfg_get(video_cfg, "video_decoder", "opencv"))
374
+ if args.video_pixel_cache_dtype == "auto":
375
+ video_pixel_cache_dtype = (
376
+ "fp16"
377
+ if args.video_clip and bool(cfg_get(video_cfg, "use_omni_hidden_cache", False))
378
+ else "none"
379
+ )
380
+ else:
381
+ video_pixel_cache_dtype = str(args.video_pixel_cache_dtype)
382
+ video_cfg_motion_fps = float(cfg_get(video_cfg, "motion_fps", TARGET_FPS))
383
+ video_embed_dir = "" if args.video_clip else str(cfg_get(video_cfg, "video_embed_dir", ""))
384
+ instruction = args.prompt.strip()
385
+
386
+ video_embed = None
387
+ video_embed_path = ""
388
+ if (not args.video_clip) and video_embed_dir.strip():
389
+ cache_path = video_embed_cache_path(
390
+ video_path,
391
+ video_embed_dir,
392
+ video_sample_fps=video_sample_fps,
393
+ video_size_pixels=video_size_pixels,
394
+ )
395
+ if cache_path.exists():
396
+ video_embed = load_video_embed_cache(cache_path)
397
+ video_embed_path = str(cache_path)
398
+
399
+ if args.video_clip:
400
+ _, _, source_video_fps, video_duration_sec = _probe_video(video_path)
401
+ video = None
402
+ elif video_embed is None:
403
+ video, source_video_fps, video_duration_sec = load_video_array(
404
+ video_path,
405
+ video_sample_fps=video_sample_fps,
406
+ video_size_pixels=video_size_pixels,
407
+ video_decoder=video_decoder,
408
+ )
409
+ else:
410
+ video = None
411
+ source_video_fps = float(video_embed["source_video_fps"])
412
+ video_duration_sec = float(video_embed["video_duration_sec"])
413
+
414
+ x0 = load_pose_from_motion_dir(motion_dir)
415
+ source_frames = load_source_frame_count(motion_dir)
416
+ target_frames = int(args.num_output_frames) if args.num_output_frames > 0 else int(source_frames or 0)
417
+ if args.video_clip and float(args.video_clip_motion_fps) > 0:
418
+ video_clip_motion_fps = float(args.video_clip_motion_fps)
419
+ elif args.video_clip and video_cfg_motion_fps > 0:
420
+ video_clip_motion_fps = float(video_cfg_motion_fps)
421
+ elif args.video_clip and source_frames > 0 and video_duration_sec > 0:
422
+ video_clip_motion_fps = float(source_frames) / float(video_duration_sec)
423
+ else:
424
+ video_clip_motion_fps = float(TARGET_FPS)
425
+
426
+ pred_abs, pred_delta, rollout_chunks = rollout_video_motion(
427
+ model=model,
428
+ maxmin=maxmin,
429
+ odim=odim,
430
+ device=args.device,
431
+ video=video,
432
+ video_embed=video_embed,
433
+ video_path=video_path,
434
+ instruction=instruction,
435
+ video_size_pixels=video_size_pixels,
436
+ video_sample_fps=video_sample_fps,
437
+ video_decoder=video_decoder,
438
+ video_pixel_cache_dtype=video_pixel_cache_dtype,
439
+ video_clip_mode=bool(args.video_clip),
440
+ video_clip_motion_fps=float(video_clip_motion_fps),
441
+ normalize_state=bool(getattr(maxmin, "normalize_state", False)),
442
+ textop_actions=textop_actions,
443
+ decode_textop_joint_from_delta=decode_textop_joint_from_delta,
444
+ num_past_frames=num_past_frames,
445
+ use_future_progress_embedding=use_future_progress_embedding,
446
+ enable_rtc=enable_rtc,
447
+ action_exec_horizon=action_exec_horizon,
448
+ rtc_max_delay=rtc_max_delay,
449
+ num_inference_steps=int(args.num_inference_steps),
450
+ target_frames=target_frames,
451
+ init_abs_state=x0,
452
+ )
453
+
454
+ motion_name = sanitize_motion_name(motion_dir.name)
455
+ sample_dir = output_dir / motion_name
456
+ sample_dir.mkdir(parents=True, exist_ok=True)
457
+
458
+ joint_pos = pred_abs[:, :29]
459
+ body_pos = pred_abs[:, 29:32]
460
+ body_quat = pred_abs[:, 32:36]
461
+
462
+ dt = 1.0 / float(TARGET_FPS)
463
+ joint_vel = finite_difference(joint_pos, dt)
464
+ body_lin_vel = finite_difference(body_pos, dt)
465
+ body_ang_vel = quaternion_angular_velocity(body_quat, dt)
466
+
467
+ write_csv(sample_dir / "joint_pos.csv", joint_pos, [f"joint_{i}" for i in range(29)])
468
+ write_csv(sample_dir / "joint_vel.csv", joint_vel, [f"joint_vel_{i}" for i in range(29)])
469
+ write_csv(sample_dir / "body_pos.csv", body_pos, ["body_0_x", "body_0_y", "body_0_z"])
470
+ write_csv(sample_dir / "body_quat.csv", body_quat, ["body_0_w", "body_0_x", "body_0_y", "body_0_z"])
471
+ write_csv(sample_dir / "body_lin_vel.csv", body_lin_vel, ["body_0_vel_x", "body_0_vel_y", "body_0_vel_z"])
472
+ write_csv(sample_dir / "body_ang_vel.csv", body_ang_vel, ["body_0_angvel_x", "body_0_angvel_y", "body_0_angvel_z"])
473
+ np.save(sample_dir / "motion.npy", pred_abs)
474
+
475
+ if args.export_delta:
476
+ write_csv(sample_dir / "joint_pos_delta.csv", pred_delta[:, :29], [f"joint_{i}" for i in range(29)])
477
+ write_csv(sample_dir / "body_pos_delta.csv", pred_delta[:, 29:32], ["body_0_x", "body_0_y", "body_0_z"])
478
+ write_csv(sample_dir / "body_quat_delta.csv", pred_delta[:, 32:36], ["body_0_w", "body_0_x", "body_0_y", "body_0_z"])
479
+ np.save(sample_dir / "motion_delta.npy", pred_delta)
480
+
481
+ write_metadata(sample_dir / "metadata.txt", motion_name, int(pred_abs.shape[0]))
482
+ write_video_info(
483
+ sample_dir / "info.txt",
484
+ video_decoder=video_decoder,
485
+ source_motion_dir=motion_dir,
486
+ video_path=video_path,
487
+ instruction=instruction,
488
+ initial_pose_source=str(motion_dir),
489
+ frames=int(pred_abs.shape[0]),
490
+ textop_actions=textop_actions,
491
+ num_past_frames=int(num_past_frames),
492
+ enable_rtc=enable_rtc,
493
+ action_exec_horizon=int(action_exec_horizon),
494
+ num_inference_steps=int(args.num_inference_steps),
495
+ video_sample_fps=float(video_sample_fps),
496
+ video_size_pixels=int(video_size_pixels),
497
+ video_pixel_cache_dtype=video_pixel_cache_dtype,
498
+ video_embed_path=video_embed_path,
499
+ source_video_fps=float(source_video_fps),
500
+ video_duration_sec=float(video_duration_sec),
501
+ full_video=not bool(args.video_clip),
502
+ video_config_key=str(args.video_config_key),
503
+ video_clip_motion_fps=float(video_clip_motion_fps),
504
+ )
505
+
506
+ for src_name, dst_name in (("info.txt", "info_source.txt"), ("metadata.txt", "metadata_source.txt")):
507
+ src_path = motion_dir / src_name
508
+ if src_path.exists():
509
+ shutil.copy2(src_path, sample_dir / dst_name)
510
+
511
+ sample_meta = {
512
+ "run_dir": str(run_dir),
513
+ "ckpt_step": ckpt_step,
514
+ "source_motion_dir": str(motion_dir),
515
+ "video_path": str(video_path),
516
+ "instruction": instruction,
517
+ "output_dir": str(sample_dir),
518
+ "num_output_frames": int(pred_abs.shape[0]),
519
+ "rollout_chunks": int(rollout_chunks),
520
+ "textop_actions": textop_actions,
521
+ "decode_textop_joint_from_delta": decode_textop_joint_from_delta,
522
+ "num_past_frames": int(num_past_frames),
523
+ "enable_rtc": enable_rtc,
524
+ "action_exec_horizon": int(action_exec_horizon),
525
+ "video_sample_fps": float(video_sample_fps),
526
+ "video_size_pixels": int(video_size_pixels),
527
+ "video_pixel_cache_dtype": video_pixel_cache_dtype,
528
+ "video_decoder": video_decoder,
529
+ "video_embed_path": video_embed_path,
530
+ "source_video_fps": float(source_video_fps),
531
+ "video_duration_sec": float(video_duration_sec),
532
+ "full_video": not bool(args.video_clip),
533
+ "video_config_key": str(args.video_config_key),
534
+ "video_clip_motion_fps": float(video_clip_motion_fps),
535
+ "use_video_instruction": bool(cfg_get(video_cfg, "use_video_instruction", False)),
536
+ "use_audio_in_video": False,
537
+ }
538
+ (sample_dir / "infer_meta.json").write_text(json.dumps(sample_meta, ensure_ascii=False, indent=2), encoding="utf-8")
539
+ (output_dir / "manifest.json").write_text(
540
+ json.dumps({"samples": [sample_meta]}, ensure_ascii=False, indent=2),
541
+ encoding="utf-8",
542
+ )
543
+ print(f"[OK] video sample -> {sample_dir}")
544
+ print(f"[DONE] wrote manifest: {output_dir / 'manifest.json'}")
545
+
546
+
547
+ if __name__ == "__main__":
548
+ main()
scripts/infer/third_party_checkpoint_probe.py ADDED
@@ -0,0 +1,434 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/usr/bin/env python3
2
+ """Fixed train-only, closed-loop probes with durable prediction evidence.
3
+
4
+ Called synchronously at a source boundary by the third-party trainer. No GT
5
+ pose after frame zero is fed to the model; no input waveform is re-mixed.
6
+ """
7
+ from __future__ import annotations
8
+
9
+ import argparse
10
+ import contextlib
11
+ import copy
12
+ import hashlib
13
+ import json
14
+ import os
15
+ from pathlib import Path
16
+ import random
17
+ import shutil
18
+ import subprocess
19
+ import sys
20
+ import time
21
+ import traceback
22
+ from typing import Any, Iterator, Sequence
23
+ import wave
24
+
25
+ import numpy as np
26
+ import torch
27
+
28
+ PROJECT_ROOT = Path(__file__).resolve().parents[2]
29
+ FPS = 50
30
+ RATE = 16000
31
+ PREVIEW_FPS = 25
32
+ PREVIEW_WIDTH = 320
33
+ PREVIEW_HEIGHT = 240
34
+ PREVIEW_CPU_THREADS = 4
35
+ PROFILES = ("single_short", "multi_single", "multi_babble", "mixed_short",
36
+ "continuous", "same_voice_aside")
37
+
38
+
39
+ def _sha256(path: Path) -> str:
40
+ digest = hashlib.sha256()
41
+ with path.open("rb") as handle:
42
+ for chunk in iter(lambda: handle.read(1024 * 1024), b""):
43
+ digest.update(chunk)
44
+ return digest.hexdigest()
45
+
46
+
47
+ def _write_json(path: Path, value: Any) -> None:
48
+ temporary = path.with_suffix(path.suffix + ".tmp")
49
+ temporary.write_text(json.dumps(value, indent=2, ensure_ascii=False,
50
+ allow_nan=False) + "\n", encoding="utf-8")
51
+ os.replace(temporary, path)
52
+
53
+
54
+ def validate_probe_recipes(recipes: Sequence[dict[str, Any]], *, probe_scope: str = "checkpoint") -> None:
55
+ ids = [str(row["session_id"]) for row in recipes]
56
+ if probe_scope not in {"checkpoint", "smoke"}:
57
+ raise ValueError("Unknown probe scope")
58
+ if any(row.get("split") != "train" for row in recipes):
59
+ raise ValueError("Checkpoint probes may only use the selected training sessions")
60
+ if probe_scope == "smoke":
61
+ if not 1 <= len(ids) <= 12 or len(set(ids)) != len(ids):
62
+ raise ValueError("Smoke probes require 1–12 distinct train sessions")
63
+ if any(row.get("session_type") not in PROFILES for row in recipes):
64
+ raise ValueError("Unknown third-party smoke profile")
65
+ return
66
+ if len(ids) != 12 or len(set(ids)) != 12:
67
+ raise ValueError("Checkpoint probes require 12 distinct fixed train sessions")
68
+ for profile in PROFILES:
69
+ rows = [row for row in recipes if row.get("session_type") == profile]
70
+ if len(rows) != 2:
71
+ raise ValueError(f"Checkpoint probes require exactly two sessions for {profile}")
72
+ languages = {row["background_assignment"]["language"] for row in rows}
73
+ if languages != {"zh", "en"}:
74
+ raise ValueError(f"Checkpoint probes need zh/en backgrounds for {profile}")
75
+
76
+
77
+ def load_frozen_probe(production_root: Path, recipe: dict[str, Any]) -> tuple[
78
+ np.ndarray, dict[str, np.ndarray], Path]:
79
+ """Hash-verify and decode PCM directly; never normalize or re-sample."""
80
+ audio_path = (production_root / recipe["audio"]["path"]).resolve()
81
+ timeline_path = (production_root / recipe["timeline"]["path"]).resolve()
82
+ for path, expected in ((audio_path, recipe["audio"]["sha256"]),
83
+ (timeline_path, recipe["timeline"]["file_sha256"])):
84
+ if _sha256(path) != expected:
85
+ raise RuntimeError(f"Frozen probe input hash changed: {path}")
86
+ with wave.open(str(audio_path), "rb") as source:
87
+ if (source.getnchannels(), source.getsampwidth(), source.getframerate()) != (1, 2, RATE):
88
+ raise ValueError("Frozen probe audio must be mono 16 kHz PCM16")
89
+ if source.getnframes() != int(recipe["audio"]["sample_count"]):
90
+ raise ValueError("Frozen audio length no longer matches its recipe")
91
+ waveform = np.frombuffer(source.readframes(source.getnframes()), dtype="<i2").astype(np.float32)
92
+ waveform /= np.float32(32768.0)
93
+ with np.load(timeline_path, allow_pickle=False) as stored:
94
+ timeline = {key: stored[key].copy() for key in ("states", "control_label")}
95
+ states, labels = timeline["states"], timeline["control_label"]
96
+ if states.ndim != 2 or states.shape[1] != 36 or len(states) < 2:
97
+ raise ValueError("Invalid frozen GT states")
98
+ if labels.shape != (len(states),) or not np.isin(labels, (0, 1, 2)).all():
99
+ raise ValueError("Invalid frozen GT control labels")
100
+ if not np.isfinite(states).all():
101
+ raise ValueError("Non-finite frozen GT states")
102
+ return waveform, timeline, audio_path
103
+
104
+
105
+ def score_prediction(motion: np.ndarray, metadata: dict[str, Any],
106
+ timeline: dict[str, np.ndarray], recipe: dict[str, Any]) -> dict[str, Any]:
107
+ """Report pose errors and control decisions separately, with explicit alignment."""
108
+ predicted = np.asarray(motion, dtype=np.float64)
109
+ states = np.asarray(timeline["states"], dtype=np.float64)
110
+ labels = np.asarray(timeline["control_label"], dtype=np.int64)
111
+ if predicted.ndim != 2 or predicted.shape[1] != 36 or not np.isfinite(predicted).all():
112
+ raise ValueError("Non-finite or malformed model prediction")
113
+ pose_count = min(len(predicted), len(states) - 1)
114
+ if pose_count <= 0:
115
+ raise ValueError("No aligned predicted/GT poses")
116
+ # Training actions at t predict t+1, while compact states store t.
117
+ delta = predicted[:pose_count] - states[1:pose_count + 1]
118
+ pred_quat = predicted[:pose_count, 32:36]
119
+ gt_quat = states[1:pose_count + 1, 32:36]
120
+ pred_norm = np.linalg.norm(pred_quat, axis=1)
121
+ gt_norm = np.linalg.norm(gt_quat, axis=1)
122
+ if np.any(pred_norm <= 1e-8) or np.any(gt_norm <= 1e-8):
123
+ raise ValueError("Zero root quaternion in predicted/GT pose")
124
+ quat_similarity = np.abs(np.sum(pred_quat * gt_quat, axis=1) / (pred_norm * gt_norm))
125
+ count = min(len(predicted), len(labels))
126
+ controls = np.full(len(predicted), -1, dtype=np.int64)
127
+ events = metadata["control_events"]
128
+ for index, event in enumerate(events):
129
+ start = int(event["output_frame"])
130
+ end = int(events[index + 1]["output_frame"]) if index + 1 < len(events) else len(predicted)
131
+ control_id = int(event["predicted_id"])
132
+ if not 0 <= start < end <= len(predicted) or control_id not in (0, 1, 2):
133
+ raise ValueError("Invalid prediction control timeline")
134
+ controls[start:end] = control_id
135
+ if np.any(controls < 0):
136
+ raise ValueError("Prediction control events do not cover the trajectory")
137
+ controls = controls[:count]
138
+ gt = labels[:count]
139
+ confusion = np.zeros((3, 3), dtype=np.int64)
140
+ np.add.at(confusion, (gt, controls), 1)
141
+ # Background-only is NOT synonymous with idle: a valid earlier command can
142
+ # still be executing. False activation is scored only when GT is non-motion.
143
+ sample_starts = np.arange(count) * (RATE // FPS)
144
+ background = np.zeros(count, dtype=bool)
145
+ target = np.zeros(count, dtype=bool)
146
+ for event in recipe.get("interference_events", []):
147
+ background |= ((sample_starts < int(event["end_sample"])) &
148
+ (sample_starts + RATE // FPS > int(event["start_sample"])))
149
+ for event in recipe.get("target_speech_events", []):
150
+ target |= ((sample_starts < int(event["end_sample"])) &
151
+ (sample_starts + RATE // FPS > int(event["start_sample"])))
152
+ background_nonmotion = background & ~target & (gt != 1)
153
+ no_motion = gt != 1
154
+ motion_gt = gt == 1
155
+ def rate(values: np.ndarray, mask: np.ndarray) -> float | None:
156
+ return float(values[mask].mean()) if mask.any() else None
157
+ return {
158
+ "pose_alignment": "predicted[t] compared with frozen state[t+1]; no DTW/realignment",
159
+ "control_alignment": "control predicted at t compared with frozen control_label[t]",
160
+ "pose_scored_frames": pose_count, "control_scored_frames": count,
161
+ "prediction_frames": len(predicted),
162
+ "unscored_prediction_tail_frames": len(predicted) - pose_count,
163
+ "joint_mae_radians": float(np.abs(delta[:, :29]).mean()),
164
+ "root_position_rmse_metres": float(np.sqrt(np.square(delta[:, 29:32]).sum(axis=1).mean())),
165
+ "root_rotation_mae_radians": float((2 * np.arccos(np.clip(quat_similarity, 0, 1))).mean()),
166
+ "control_accuracy": float((controls == gt).mean()),
167
+ "control_confusion_rows_gt_columns_prediction": confusion.tolist(),
168
+ "control_order": ["listen", "motion", "idle"],
169
+ "nonmotion_gt_frames": int(no_motion.sum()),
170
+ "false_motion_rate": rate(controls == 1, no_motion),
171
+ "motion_gt_frames": int(motion_gt.sum()),
172
+ "missed_motion_rate": rate(controls != 1, motion_gt),
173
+ "background_only_nonmotion_gt_frames": int(background_nonmotion.sum()),
174
+ "background_only_false_motion_rate": rate(controls == 1, background_nonmotion),
175
+ "root_height_min_metres": float(predicted[:, 31].min()),
176
+ "root_height_max_metres": float(predicted[:, 31].max()),
177
+ "joint_step_rms_radians": float(np.sqrt(np.square(np.diff(predicted[:, :29], axis=0)).mean())),
178
+ "interpretation": "Train memorization/robustness diagnostic, not held-out generalization or physics success.",
179
+ }
180
+
181
+
182
+ @contextlib.contextmanager
183
+ def preserve_training_state(model: Any, seed: int) -> Iterator[None]:
184
+ """Probe must not alter dropout flags, RNG streams, or training scheduler."""
185
+ module_flags = [(module, module.training) for module in model.modules()]
186
+ python_rng, numpy_rng = random.getstate(), np.random.get_state()
187
+ torch_rng = torch.get_rng_state()
188
+ cuda_rng = torch.cuda.get_rng_state_all() if torch.cuda.is_initialized() else None
189
+ old_scheduler = getattr(model, "noise_scheduler", None)
190
+ try:
191
+ if old_scheduler is not None:
192
+ model.noise_scheduler = copy.deepcopy(old_scheduler)
193
+ random.seed(seed)
194
+ np.random.seed(seed % (2 ** 32))
195
+ torch.manual_seed(seed)
196
+ model.eval()
197
+ with torch.inference_mode():
198
+ yield
199
+ finally:
200
+ for module, training in module_flags:
201
+ module.training = training
202
+ if old_scheduler is not None:
203
+ model.noise_scheduler = old_scheduler
204
+ random.setstate(python_rng)
205
+ np.random.set_state(numpy_rng)
206
+ torch.set_rng_state(torch_rng)
207
+ if cuda_rng is not None:
208
+ torch.cuda.set_rng_state_all(cuda_rng)
209
+
210
+
211
+ @contextlib.contextmanager
212
+ def inference_bindings(model: Any, launch_config: Any, vlm_processor: Any) -> Iterator[None]:
213
+ """Training keeps processor/scheduler on the trainer, unlike loaded inference."""
214
+ from diffusers.schedulers.scheduling_flow_match_euler_discrete import FlowMatchEulerDiscreteScheduler
215
+
216
+ missing = object()
217
+ names = ("device", "vlm_processor", "noise_scheduler")
218
+ previous = {name: getattr(model, name, missing) for name in names}
219
+ cfg = getattr(launch_config, "model", None)
220
+ if str(getattr(cfg, "noise_scheduler", "flow")) != "flow":
221
+ raise ValueError("Closed-loop third-party probes require the flow scheduler")
222
+ processor = vlm_processor if vlm_processor is not None else getattr(model, "vlm_processor", None)
223
+ if processor is None:
224
+ raise ValueError("In-process checkpoint probe requires trainer.vlm_processor")
225
+ try:
226
+ model.device = next(model.parameters()).device
227
+ model.vlm_processor = processor
228
+ model.noise_scheduler = FlowMatchEulerDiscreteScheduler(
229
+ num_train_timesteps=int(getattr(cfg, "train_diffusion_steps", 1000)))
230
+ yield
231
+ finally:
232
+ for name, value in previous.items():
233
+ if value is missing:
234
+ if hasattr(model, name):
235
+ delattr(model, name)
236
+ else:
237
+ setattr(model, name, value)
238
+
239
+
240
+ def _run(command: Sequence[str], log: Path, **kwargs: Any) -> None:
241
+ with log.open("a", encoding="utf-8") as handle:
242
+ result = subprocess.run(list(command), stdout=handle, stderr=subprocess.STDOUT,
243
+ check=False, timeout=1800, **kwargs)
244
+ if result.returncode:
245
+ raise RuntimeError(f"Probe render failed (exit {result.returncode}); see {log}")
246
+
247
+
248
+ def render_prediction(sample_dir: Path, audio_path: Path) -> Path:
249
+ """Low-cost CPU review video; the underlying predicted CSV remains 50 Hz."""
250
+ from scripts.data.render_hfd_third_party_gt import (
251
+ GMR_PYTHON, GMR_XML, RENDER_ROOT, _render_environment,
252
+ )
253
+ from scripts.infer.mux_full_duplex_audio import probe as probe_media, stream
254
+ from scripts.third_party_temp import project_temp_directory
255
+
256
+ environment = _render_environment("cpu", cpu_threads=PREVIEW_CPU_THREADS)
257
+ scratch = str(project_temp_directory(PROJECT_ROOT))
258
+ environment.update({name: scratch for name in ("TMPDIR", "TMP", "TEMP", "SONIC_RENDER_TMPDIR")})
259
+ meta_path = sample_dir / "infer_meta.json"
260
+ metadata = json.loads(meta_path.read_text(encoding="utf-8"))
261
+ metadata["review_render"] = {
262
+ "motion_fps": FPS, "video_fps": PREVIEW_FPS,
263
+ "width": PREVIEW_WIDTH, "height": PREVIEW_HEIGHT,
264
+ "cpu_threads": PREVIEW_CPU_THREADS,
265
+ "purpose": "Checkpoint screening preview; full 50Hz predicted poses and metrics retained",
266
+ "audio_policy": "Complete original mixed WAV, no time stretch or offset",
267
+ }
268
+ _write_json(meta_path, metadata)
269
+ log = sample_dir / "render.log"
270
+ _run([str(GMR_PYTHON), str(RENDER_ROOT / "scripts/batch_render_reference_motion_mujoco.py"),
271
+ "--motion-dir", str(sample_dir), "--xml-path", str(GMR_XML),
272
+ "--source-fps", str(FPS), "--target-fps", str(PREVIEW_FPS),
273
+ "--width", str(PREVIEW_WIDTH), "--height", str(PREVIEW_HEIGHT),
274
+ "--dof-order", "isaaclab", "--camera-distance", "2.5"], log,
275
+ cwd=sample_dir, env=environment)
276
+ # The shared renderer calls every CSV input 'reference'; rename it here so
277
+ # users do not confuse the actual model prediction with the dataset's GT.
278
+ silent = sample_dir / "prediction_silent.mp4"
279
+ (sample_dir / "reference_motion.mp4").replace(silent)
280
+ media = probe_media(silent)
281
+ video_stream = stream(media, "video")
282
+ with wave.open(str(audio_path), "rb") as source:
283
+ audio_seconds = source.getnframes() / source.getframerate()
284
+ video_seconds = float(video_stream.get("duration") or media["format"]["duration"])
285
+ if (video_stream["width"], video_stream["height"]) != (PREVIEW_WIDTH, PREVIEW_HEIGHT):
286
+ raise RuntimeError("Checkpoint preview resolution drifted")
287
+ if video_seconds < audio_seconds - 1e-6 or video_seconds - audio_seconds > 1 / PREVIEW_FPS + 1e-6:
288
+ raise RuntimeError("Checkpoint preview would truncate or extend the original audio timeline")
289
+ video = sample_dir / "prediction_with_audio.mp4"
290
+ _run([sys.executable, str(PROJECT_ROOT / "scripts/infer/mux_full_duplex_audio.py"),
291
+ "--video", str(silent), "--audio", str(audio_path), "--output", str(video),
292
+ "--infer-meta", str(sample_dir / "infer_meta.json"), "--audio-is-timeline",
293
+ "--expected-fps", str(PREVIEW_FPS)], log, env=environment)
294
+ _run(["ffmpeg", "-v", "error", "-xerror", "-i", str(video),
295
+ "-f", "null", "-"], log, env=environment)
296
+ return video
297
+
298
+
299
+ def _aggregate(rows: Sequence[dict[str, Any]]) -> dict[str, Any]:
300
+ keys = ("joint_mae_radians", "root_position_rmse_metres", "control_accuracy",
301
+ "root_rotation_mae_radians",
302
+ "false_motion_rate", "missed_motion_rate", "background_only_false_motion_rate")
303
+ result: dict[str, Any] = {"averaging": "unweighted mean over available session metrics"}
304
+ for key in keys:
305
+ values = [row["metrics"][key] for row in rows if row["metrics"].get(key) is not None]
306
+ result[key] = float(np.mean(values)) if values else None
307
+ result[key + "_sessions"] = len(values)
308
+ return result
309
+
310
+
311
+ def run_checkpoint_probes(model: Any, launch_config: Any, *, production_root: str | Path,
312
+ recipes: Sequence[dict[str, Any]], output_root: str | Path,
313
+ global_step: int, seed: int = 260930,
314
+ num_inference_steps: int = 10,
315
+ probe_scope: str = "checkpoint",
316
+ vlm_processor: Any = None) -> dict[str, Any]:
317
+ """Run all fixed 12 predictions; persist partial failure and re-raise errors.
318
+
319
+ The trainer must call only at a source boundary, and pass recipes from its
320
+ frozen 300-session selection. output_root is normally run/checkpoint_probes.
321
+ """
322
+ from scripts.infer import full_duplex_audio_to_motion as infer
323
+
324
+ validate_probe_recipes(recipes, probe_scope=probe_scope)
325
+ if getattr(model, "control_condition_inference_mode", "hard") != "hard":
326
+ raise ValueError("Third-party checkpoint probes require own hard control conditioning")
327
+ if int(global_step) < 0 or int(num_inference_steps) <= 0:
328
+ raise ValueError("Invalid probe step/inference steps")
329
+ production_root = Path(production_root).resolve()
330
+ output_root = Path(output_root).resolve()
331
+ if not output_root.is_relative_to(PROJECT_ROOT.resolve()):
332
+ raise ValueError("Probe outputs must remain inside HFD_lzk")
333
+ step_dir = output_root / f"step_{int(global_step):08d}"
334
+ step_dir.mkdir(parents=True, exist_ok=False)
335
+ started = time.monotonic()
336
+ report: dict[str, Any] = {
337
+ "format": "hfd_third_party_checkpoint_probe_v1", "status": "running",
338
+ "global_step": int(global_step), "seed": int(seed), "evaluation_split": "train_probe",
339
+ "probe_scope": probe_scope,
340
+ "fixed_session_ids": [row["session_id"] for row in recipes],
341
+ "input_policy": "Exact frozen mixed PCM16; no remix, resampling, gain change, or video input",
342
+ "rollout_policy": "Own hard control and predicted state/RTC; only initial GT pose supplied",
343
+ "tail_policy": "Final 0.2s uses last predicted chunk tail, not a GT or stationary hold",
344
+ "render_backend": "mesa_llvmpipe_cpu", "motion_fps": FPS,
345
+ "review_video_fps": PREVIEW_FPS, "review_video_resolution": [PREVIEW_WIDTH, PREVIEW_HEIGHT],
346
+ "render_cpu_threads": PREVIEW_CPU_THREADS, "samples": [],
347
+ }
348
+ _write_json(step_dir / "report.json", report)
349
+ try:
350
+ with preserve_training_state(model, seed), inference_bindings(model, launch_config, vlm_processor):
351
+ for index, recipe in enumerate(recipes, 1):
352
+ session_id = str(recipe["session_id"])
353
+ sample_dir = step_dir / f"{index:02d}_{recipe['session_type']}_{session_id[:12]}"
354
+ sample_dir.mkdir()
355
+ waveform, timeline, audio_path = load_frozen_probe(production_root, recipe)
356
+ print(f"[train_probe] step={global_step} sample={index}/{len(recipes)} session={session_id}", flush=True)
357
+ motion, metadata = infer.rollout_full_duplex(
358
+ model=model, background_waveforms=[], waveform=waveform,
359
+ prompt=infer.DEFAULT_COMMAND_PROMPT, source_id=f"train_probe:{session_id}",
360
+ init_abs=timeline["states"][0].copy(), launch_config=launch_config,
361
+ motion_frames=max(int(getattr(launch_config.data, "turn_transition_frames", 50)), 50) + 300,
362
+ num_inference_steps=int(num_inference_steps), respect_control_token=True,
363
+ max_listen_tail_steps=1, initial_idle_steps=0,
364
+ seed=seed + index, frozen_audio_timeline=True,
365
+ )
366
+ expected = int(round(len(waveform) * FPS / RATE))
367
+ if motion.shape != (expected, 36):
368
+ raise RuntimeError("Frozen probe prediction does not cover its exact audio timeline")
369
+ metrics = score_prediction(motion, metadata, timeline, recipe)
370
+ metadata.update({"session_id": session_id, "global_step": int(global_step),
371
+ "frozen_audio_path": str(audio_path),
372
+ "frozen_audio_sha256": recipe["audio"]["sha256"],
373
+ "artifact_role": "model_prediction_not_GT"})
374
+ infer.export_motion(sample_dir, motion)
375
+ _write_json(sample_dir / "infer_meta.json", metadata)
376
+ _write_json(sample_dir / "metrics.json", metrics)
377
+ _write_json(sample_dir / "session_recipe.json", recipe)
378
+ video = render_prediction(sample_dir, audio_path)
379
+ row = {"session_id": session_id, "profile": recipe["session_type"],
380
+ "background_language": recipe["background_assignment"]["language"],
381
+ "background_sir_db": recipe.get("mix_metadata", {}).get("measured_background_sir_db"),
382
+ "metrics": metrics, "video": str(video), "video_sha256": _sha256(video),
383
+ "motion_sha256": _sha256(sample_dir / "motion.npy"),
384
+ "full_video_decode_passed": True}
385
+ report["samples"].append(row)
386
+ _write_json(step_dir / "report.json", report)
387
+ report["status"] = "complete"
388
+ report["aggregate"] = _aggregate(report["samples"])
389
+ except BaseException as error:
390
+ report["status"] = "failed"
391
+ report["error"] = f"{type(error).__name__}: {error}"
392
+ (step_dir / "failure.txt").write_text(traceback.format_exc(), encoding="utf-8")
393
+ raise
394
+ finally:
395
+ report["elapsed_seconds"] = time.monotonic() - started
396
+ _write_json(step_dir / "report.json", report)
397
+ _write_json(output_root / "latest.json", {"report": str(step_dir / "report.json"),
398
+ "status": report["status"], "global_step": global_step})
399
+ with (output_root / "history.jsonl").open("a", encoding="utf-8") as handle:
400
+ handle.write(json.dumps({"global_step": global_step, "status": report["status"],
401
+ "report": str(step_dir / "report.json"),
402
+ "elapsed_seconds": report["elapsed_seconds"],
403
+ "aggregate": report.get("aggregate", {})}, ensure_ascii=False) + "\n")
404
+ lines = ["# Checkpoint 训练集闭环记录", "", f"step={global_step},状态:{report['status']}。",
405
+ "这些是模型预测,不是 GT;训练集结果不代表泛化能力。", "",
406
+ "审核视频为 320×240 / 25 fps;完整预测轨迹和指标保留 50 Hz,原始混音时间线不变。", "",
407
+ "| 序号 | 类型 | 背景语言 | 控制准确率 | 视频 |", "|---|---|---|---:|---|"]
408
+ for index, row in enumerate(report["samples"], 1):
409
+ relative_video = Path(row["video"]).relative_to(step_dir)
410
+ lines.append(f"| {index:02d} | {row['profile']} | {row['background_language']} | "
411
+ f"{row['metrics']['control_accuracy']:.3f} | [模型预测]({relative_video}) |")
412
+ if report.get("error"):
413
+ lines.extend(["", "失败原因见 failure.txt;未生成的样本不得视为通过。"])
414
+ (step_dir / "README.md").write_text("\n".join(lines) + "\n", encoding="utf-8")
415
+ return report
416
+
417
+
418
+ def check_renderer() -> dict[str, Any]:
419
+ from scripts.data.render_hfd_third_party_gt import GMR_PYTHON, GMR_XML, RENDER_ROOT
420
+ required = (GMR_PYTHON, GMR_XML, RENDER_ROOT / "scripts/batch_render_reference_motion_mujoco.py")
421
+ missing = [str(path) for path in required if not path.is_file()]
422
+ missing += [name for name in ("ffmpeg", "ffprobe") if not shutil.which(name)]
423
+ if missing:
424
+ raise FileNotFoundError(f"Missing checkpoint video renderer requirements: {missing}")
425
+ return {"paths_present": True, "backend": "cpu", "render_execution_verified": False}
426
+
427
+
428
+ if __name__ == "__main__":
429
+ if str(PROJECT_ROOT) not in sys.path:
430
+ sys.path.insert(0, str(PROJECT_ROOT))
431
+ parser = argparse.ArgumentParser(description=__doc__)
432
+ parser.add_argument("--check-renderer", action="store_true", required=True)
433
+ parser.parse_args()
434
+ print(json.dumps(check_renderer(), ensure_ascii=False))
scripts/infer/third_party_scale_metrics.py ADDED
@@ -0,0 +1,172 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/usr/bin/env python3
2
+ """GPU-0 scale-stage closed-loop metrics on separately frozen train/validation."""
3
+ from __future__ import annotations
4
+
5
+ import argparse
6
+ import hashlib
7
+ import json
8
+ import os
9
+ from pathlib import Path
10
+ import subprocess
11
+ import sys
12
+ import time
13
+
14
+ PROJECT = Path(__file__).resolve().parents[2]
15
+ sys.path[:0] = [str(PROJECT), str(PROJECT / "src")]
16
+ from scripts.third_party_strategy_plan import write_json
17
+ from scripts.infer.third_party_metrics import aggregate, local
18
+ from scripts.third_party_scale_plan import SEED, safe_relative_path
19
+
20
+
21
+ def read(path):
22
+ return json.loads(Path(path).read_text())
23
+
24
+
25
+ def sample_seed(sid):
26
+ return int(hashlib.sha256(f"{SEED}:rollout:{sid}".encode()).hexdigest()[:8], 16)
27
+
28
+
29
+ def recipes_for(manifest, ids):
30
+ from psi.data.seed.third_party_frozen_dataset import read_jsonl, sha256_file
31
+ selection = read(manifest)
32
+ split = selection.get("split")
33
+ if (selection.get("scaleup") is not True or split not in ("train", "validation")
34
+ or len(set(ids)) != len(ids) or not set(ids) <= set(selection["session_ids"])):
35
+ raise ValueError("Invalid scale evaluation split/IDs")
36
+ root = local(selection["production_root"])
37
+ path = root / "manifests/session_recipes.jsonl"
38
+ if sha256_file(path) != selection["recipe_manifest_sha256"]:
39
+ raise ValueError("Evaluation recipe manifest changed")
40
+ wanted, found = set(ids), {}
41
+ for row in read_jsonl(path):
42
+ if row["session_id"] in wanted:
43
+ if row["session_id"] in found or row["split"] != split:
44
+ raise ValueError("Evaluation crossed production split")
45
+ for name in ("audio", "timeline"):
46
+ safe_relative_path(root, row[name]["path"])
47
+ found[row["session_id"]] = row
48
+ if set(found) != wanted:
49
+ raise ValueError("Missing evaluation recipe")
50
+ return root, found, split
51
+
52
+
53
+ def evaluate(source_run, step, manifest, output, *, full=False, workers=4, include_videos=False):
54
+ if os.environ.get("CUDA_VISIBLE_DEVICES") != "0" or workers not in (1, 2, 4):
55
+ raise RuntimeError("Use the managed single-GPU-0 scale launcher")
56
+ source_run, manifest, output = map(local, (source_run, manifest, output))
57
+ if output.exists() or not output.is_relative_to(PROJECT / "runs"):
58
+ raise ValueError("Fresh local evaluation directory required")
59
+ selection = read(manifest)
60
+ ids = list(selection["session_ids"] if full else selection["evaluation_session_ids"])
61
+ if include_videos:
62
+ ids.extend(sid for sid in selection["video_session_ids"] if sid not in ids)
63
+ if not ids or len(set(ids)) != len(ids):
64
+ raise ValueError("Invalid fixed evaluation panel")
65
+ output.mkdir(parents=True)
66
+ (output / "samples").mkdir()
67
+ task = {"source_run": str(source_run), "step": step, "manifest": str(manifest),
68
+ "manifest_sha256": hashlib.sha256(manifest.read_bytes()).hexdigest(),
69
+ "output": str(output), "split": selection["split"], "all_ids": ids}
70
+ write_json(output / "identity.json", task)
71
+ started = time.monotonic()
72
+ children, logs = [], []
73
+ try:
74
+ for rank in range(min(workers, len(ids))):
75
+ path = output / f"worker_{rank}.json"
76
+ write_json(path, {**task, "ids": ids[rank::workers]})
77
+ log = (output / f"worker_{rank}.log").open("x"); logs.append(log)
78
+ children.append(subprocess.Popen([sys.executable, str(Path(__file__).resolve()), "--worker", str(path)],
79
+ cwd=PROJECT, stdout=log, stderr=subprocess.STDOUT))
80
+ while any(child.poll() is None for child in children):
81
+ if any(child.poll() not in (None, 0) for child in children):
82
+ raise RuntimeError(f"Scale evaluation worker failed: {output}")
83
+ time.sleep(1)
84
+ if any(child.returncode != 0 for child in children):
85
+ raise RuntimeError(f"Scale evaluation failed: {output}")
86
+ finally:
87
+ for child in children:
88
+ if child.poll() is None:
89
+ child.terminate()
90
+ for child in children:
91
+ try:
92
+ child.wait(timeout=30)
93
+ except subprocess.TimeoutExpired:
94
+ child.kill(); child.wait()
95
+ for log in logs:
96
+ log.close()
97
+ rows = [read(output / "samples" / sid / "result.json") for sid in ids]
98
+ by_id = {r["session_id"]: r for r in selection["sessions"]}
99
+ groups = {}
100
+ for field in ("session_type", "background_language", "background_high"):
101
+ groups[field] = {value: {"count": len(subset), **aggregate(subset)}
102
+ for value in sorted({str(by_id[r["session_id"]][field]) for r in rows})
103
+ if (subset := [r for r in rows if str(by_id[r["session_id"]][field]) == value])}
104
+ report = {"status": "complete", "step": step, "evaluation_split": selection["split"],
105
+ "panel": "all_eligible" if full else "fixed_class_language_panel",
106
+ "count": len(rows), "samples": rows, "aggregate": aggregate(rows), "groups": groups,
107
+ "elapsed_seconds": time.monotonic() - started, "workers": workers, "rendered_videos": 0,
108
+ "input_policy": "original full mixed WAV; no GT after initial pose; no optimizer updates",
109
+ "seed_policy": "stable per session, independent of workers/checkpoint/panel size"}
110
+ panel = set(selection["evaluation_session_ids"])
111
+ fixed = [r for r in rows if r["session_id"] in panel]
112
+ report["fixed_panel_comparison"] = {"count": len(fixed), "aggregate": aggregate(fixed)}
113
+ write_json(output / "report.json", report)
114
+ return report
115
+
116
+
117
+ def worker(path):
118
+ import numpy as np
119
+ import torch
120
+ from scripts.third_party_temp import configure_project_temp
121
+ from scripts.infer import full_duplex_audio_to_motion as infer
122
+ from scripts.infer.third_party_checkpoint_probe import load_frozen_probe, score_prediction, _sha256
123
+ from psi.models.full_duplex_psi0 import FullDuplexPsi0Model
124
+ from psi.config.train.posttrain_third_party_config import DynamicLaunchConfig
125
+ if os.environ.get("CUDA_VISIBLE_DEVICES") != "0":
126
+ raise RuntimeError("Scale metrics require GPU0 isolation")
127
+ configure_project_temp(PROJECT); torch.set_num_threads(2)
128
+ task = read(local(path)); manifest = local(task["manifest"])
129
+ if hashlib.sha256(manifest.read_bytes()).hexdigest() != task["manifest_sha256"]:
130
+ raise ValueError("Metric task selection changed")
131
+ source = local(task["source_run"])
132
+ cfg = DynamicLaunchConfig.model_validate_json((source / "run_config.json").read_text())
133
+ infer.validate_contract(cfg)
134
+ model = FullDuplexPsi0Model.from_streaming_checkpoint(source, task["step"], cfg, torch.device("cuda:0"))
135
+ model.eval()
136
+ root, recipes, split = recipes_for(manifest, task["ids"])
137
+ if split != task["split"]:
138
+ raise ValueError("Metric task split changed")
139
+ with torch.inference_mode():
140
+ for sid in task["ids"]:
141
+ started = time.monotonic(); recipe = recipes[sid]
142
+ dest = local(task["output"]) / "samples" / sid; dest.mkdir()
143
+ waveform, timeline, audio = load_frozen_probe(root, recipe)
144
+ seed = sample_seed(sid)
145
+ motion, metadata = infer.rollout_full_duplex(model=model, background_waveforms=[], waveform=waveform,
146
+ prompt=infer.DEFAULT_COMMAND_PROMPT, source_id=f"scale_{split}:{sid}",
147
+ init_abs=timeline["states"][0].copy(), launch_config=cfg,
148
+ motion_frames=max(int(cfg.data.turn_transition_frames), 50) + 300,
149
+ num_inference_steps=10, respect_control_token=True, max_listen_tail_steps=1,
150
+ initial_idle_steps=0, seed=seed, frozen_audio_timeline=True)
151
+ if motion.shape != (round(len(waveform) * 50 / 16000), 36) or not np.isfinite(motion).all():
152
+ raise ValueError("Incomplete/nonfinite scale prediction")
153
+ metrics = score_prediction(motion, metadata, timeline, recipe)
154
+ metrics["interpretation"] = ("Held-out production validation; not physics success" if split == "validation"
155
+ else "Training fit diagnostic, not held-out generalization")
156
+ metadata.update(session_id=sid, global_step=task["step"], evaluation_split=split,
157
+ frozen_audio_path=str(audio), frozen_audio_sha256=recipe["audio"]["sha256"],
158
+ artifact_role="model_prediction_not_GT")
159
+ infer.export_motion(dest, motion)
160
+ for name, value in (("infer_meta", metadata), ("metrics", metrics), ("session_recipe", recipe)):
161
+ write_json(dest / f"{name}.json", value)
162
+ row = {"session_id": sid, "profile": recipe["session_type"], "split": split,
163
+ "background_language": recipe["background_assignment"]["language"], "seed": seed,
164
+ "metrics": metrics, "prediction_directory": str(dest), "motion_sha256": _sha256(dest / "motion.npy"),
165
+ "render_status": "not_requested", "inference_seconds": time.monotonic() - started}
166
+ write_json(dest / "result.json", row)
167
+ print(json.dumps({"session": sid, "split": split, "seconds": row["inference_seconds"]}), flush=True)
168
+
169
+
170
+ if __name__ == "__main__":
171
+ parser = argparse.ArgumentParser(); parser.add_argument("--worker", required=True)
172
+ worker(parser.parse_args().worker)
src/InternVLA-M1/InternVLA/__init__.py ADDED
File without changes
src/InternVLA-M1/InternVLA/config/deepseeds/deepspeed_zero2.yaml ADDED
@@ -0,0 +1,10 @@
 
 
 
 
 
 
 
 
 
 
 
1
+ compute_environment: LOCAL_MACHINE
2
+ debug: false
3
+ deepspeed_config:
4
+ deepspeed_config_file: "InternVLA/config/deepseeds/ds_config.yaml"
5
+ deepspeed_multinode_launcher: standard
6
+ zero3_init_flag: false
7
+ distributed_type: DEEPSPEED
8
+ # main_process_port: 12345
9
+ num_machines: 1
10
+ num_processes: 8
src/InternVLA-M1/InternVLA/config/deepseeds/ds_config.yaml ADDED
@@ -0,0 +1,28 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "fp16": {
3
+ "enabled": "auto",
4
+ "loss_scale": 0,
5
+ "loss_scale_window": 1000,
6
+ "initial_scale_power": 16,
7
+ "hysteresis": 2,
8
+ "min_loss_scale": 1
9
+ },
10
+ "bf16": {
11
+ "enabled": "auto"
12
+ },
13
+ "train_micro_batch_size_per_gpu": "auto",
14
+ "train_batch_size": "auto",
15
+ "gradient_accumulation_steps": 1,
16
+ "zero_optimization": {
17
+ "stage": 2,
18
+ "allgather_partitions": true,
19
+ "allgather_bucket_size": 5e8,
20
+ "reduce_scatter": true,
21
+ "reduce_bucket_size": 5e8,
22
+ "overlap_comm": true,
23
+ "contiguous_gradients": true,
24
+ "cpu_offload": false
25
+ },
26
+ "gradient_clipping": 1.0,
27
+ "steps_per_print": 10
28
+ }
src/InternVLA-M1/InternVLA/config/deepseeds/zero2.yaml ADDED
@@ -0,0 +1,21 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ compute_environment: LOCAL_MACHINE
2
+ debug: false
3
+ deepspeed_config:
4
+ deepspeed_multinode_launcher: standard
5
+ offload_optimizer_device: none
6
+ offload_param_device: none
7
+ zero3_init_flag: false
8
+ zero_stage: 2
9
+ distributed_type: DEEPSPEED
10
+ downcast_bf16: 'no'
11
+ machine_rank: 0
12
+ main_training_function: main
13
+ mixed_precision: bf16
14
+ num_machines: 1
15
+ num_processes: 8
16
+ rdzv_backend: static
17
+ same_network: true
18
+ tpu_env: []
19
+ tpu_use_cluster: false
20
+ tpu_use_sudo: false
21
+ use_cpu: false
src/InternVLA-M1/InternVLA/config/deepseeds/zero3.json ADDED
@@ -0,0 +1,28 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "fp16": {
3
+ "enabled": "auto",
4
+ "loss_scale": 0,
5
+ "loss_scale_window": 1000,
6
+ "initial_scale_power": 16,
7
+ "hysteresis": 2,
8
+ "min_loss_scale": 1
9
+ },
10
+ "bf16": {
11
+ "enabled": "auto"
12
+ },
13
+ "train_micro_batch_size_per_gpu": "auto",
14
+ "train_batch_size": "auto",
15
+ "gradient_accumulation_steps": "auto",
16
+ "zero_optimization": {
17
+ "stage": 3,
18
+ "overlap_comm": true,
19
+ "contiguous_gradients": true,
20
+ "sub_group_size": 1e9,
21
+ "reduce_bucket_size": "auto",
22
+ "stage3_prefetch_bucket_size": "auto",
23
+ "stage3_param_persistence_threshold": "auto",
24
+ "stage3_max_live_parameters": 1e9,
25
+ "stage3_max_reuse_distance": 1e9,
26
+ "stage3_gather_16bit_weights_on_model_save": true
27
+ }
28
+ }
src/InternVLA-M1/InternVLA/config/training/internvla_cotrain_custom.yaml ADDED
@@ -0,0 +1,126 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ run_id: internvla_cotrain_custom
2
+ run_root_dir: results/Checkpoints
3
+ seed: 42
4
+ trackers: [jsonl, wandb]
5
+ wandb_entity: WANDB_ENTITY
6
+ wandb_project: WANDB_PROJECT
7
+ is_debug: false
8
+
9
+ framework:
10
+ framework_py: InternVLA-M1
11
+ qwenvl: # TODO later should not use qwenvl should use vlm directly
12
+ base_vlm: ./playground/Pretrained_models/Qwen2.5-VL-3B-Instruct
13
+ attn_implementation: flash_attention_2
14
+ vl_hidden_dim: 2048 # TODO should read from internal config
15
+ dino:
16
+ dino_backbone: dinov2_vits14
17
+ layer_qformer:
18
+ qformer_end_layer: 37
19
+ qformer_start_layer: 36
20
+ num_query_tokens: 64
21
+ input_dim: 2048 # here is VLM output dimension
22
+ ouptput_dim: 768 # here is action input dimension # here has a rule that ouptput_dim = action_hidden_dim; --> top2down logic should not default match, should be modified manually
23
+ grad_scale: 0.5 # let gradient pass through this module to decay, avoid destroying VLM, # seems to affect learning
24
+
25
+ action_model:
26
+ action_model_type: DiT-B
27
+ action_hidden_dim: 768
28
+ action_dim: 7
29
+ use_ema: false
30
+ future_action_window_size: 15
31
+ past_action_window_size: 0
32
+ repeated_diffusion_steps: 8
33
+
34
+ fm_head_config:
35
+ input_embedding_dim: 1536 # Action embedding after dimension 48 * 32,and DiT's attention_head_dim * attention_heads corresponds to ActionEncoder
36
+ hidden_size: 1024 # and DiT's last projection corresponds to ActionDecoder
37
+ add_pos_embed: True
38
+ max_seq_len: 1024
39
+ action_dim: 7
40
+ future_action_window_size: 15
41
+ action_horizon: 16 # = future_action_window_size + 1 + past_action_window_size
42
+ past_action_window_size: 0
43
+ noise_beta_alpha: 1.5
44
+ noise_beta_beta: 1.0
45
+ noise_s: 0.999
46
+ num_timestep_buckets: 1000
47
+ num_inference_timesteps: 4
48
+ num_target_vision_tokens: 32
49
+ diffusion_model_cfg: # DiT Transformers parameters
50
+ attention_head_dim: 48
51
+ cross_attention_dim: 2048 # VLM dim
52
+ dropout: 0.2
53
+ final_dropout: true
54
+ interleave_self_attention: true
55
+ norm_type: "ada_norm"
56
+ num_attention_heads: 32
57
+ num_layers: 16
58
+ output_dim: 1024
59
+ positional_embeddings: null
60
+
61
+ datasets:
62
+ vlm_data:
63
+ dataset_py: vlm_datasets
64
+ dataformat: llava_json
65
+ dataset_use: asv2_conversation_en,asv2_detailed_description_en,asv2_region_captioning_en,coco_internvl_longcap_en,coco_karpathy_train_567_en,coco_negative_gpt4o_en,coco_poetry_zh,coco_rem_en_zh,cocorem_exist_yorn_en,cocotextv2_en,cocotextv2_gpt4o_en,okvqa_en,refcoco_grounding_aug_en,refcoco_grounding_en,tallyqa_coco_en,toloka_grounding_aug_en,vqav2_en,vsr_en
66
+ eval_dataset: aokvqa_cauldron_llava_format
67
+ data_flatten: false
68
+ base_interval: 2
69
+ max_pixels: 50176
70
+ min_pixels: 784
71
+ model_max_length: 2048 # seems to be 4096 here, some related content is not processed
72
+ model_type: qwen2.5vl
73
+ per_device_batch_size: 4
74
+
75
+ vla_data:
76
+ dataset_py: lerobot_datasets
77
+ data_root_dir: playground/Datasets/YOUR_DATASET_PATH
78
+ data_mix: YOUR_DATASET_NAME
79
+ action_type: delta_ee
80
+ CoT_prompt: "Your task is {instruction}. To identify the key objects for your task. Locate their bounding boxes in [x1,y1,x2,y2] format."
81
+ CoT_answer: bbox
82
+ default_image_resolution: [3, 224, 224] # this is not effective
83
+ per_device_batch_size: 16
84
+ load_all_data_for_training: true # consider how to enable eval set
85
+ obs: ["image_0"] # this is not effective
86
+ image_size: [224,224] #todo is null keep raw size # this is not effective
87
+
88
+ trainer:
89
+ epochs: 100 # decide whether to use, he and max_train_steps conflict
90
+ max_train_steps: 100000
91
+ num_warmup_steps: 5000
92
+ save_interval: 5000
93
+ eval_interval: 100
94
+ learning_rate: # you can set different lr for different modules
95
+ base: 5e-05
96
+ qwen_vl_interface: 1.0e-05
97
+ action_model: 1.0e-04
98
+ lr_scheduler_type: cosine_with_min_lr
99
+ scheduler_specific_kwargs:
100
+ min_lr: 5.0e-07
101
+ freeze_modules: ''
102
+ loss_scale:
103
+ vla: 1.0
104
+ vlm: 0.1
105
+ # pretrained_checkpoint: ./results/Checkpoints/1_need/0601_qwenact_fixqwen_32gpus_lr_1e-3_qformer_36_37/steps_100000/pytorch_model.pt
106
+ # reload_modules: 'action_model,layer_qformer' # warning: this may cause model to not converge
107
+ repeated_diffusion_steps: 4
108
+ max_grad_norm: 1.0
109
+ warmup_ratio: 0.1
110
+ weight_decay: 0.0
111
+ logging_frequency: 10
112
+ gradient_clipping: 1.0
113
+ gradient_accumulation_steps: 1
114
+
115
+ optimizer:
116
+ name: AdamW
117
+ betas: [0.9, 0.95]
118
+ eps: 1.0e-08
119
+ weight_decay: 1.0e-08
120
+
121
+ # parameters to be determined
122
+ is_resume: false
123
+ resume_epoch: null
124
+ resume_step: null
125
+ enable_gradient_checkpointing: true
126
+ enable_mixed_precision_training: true
src/InternVLA-M1/InternVLA/config/training/internvla_cotrain_humanoid.yaml ADDED
@@ -0,0 +1,128 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ run_id: humanoid_g1
2
+ run_root_dir: runs/InternVLA/Checkpoints
3
+ seed: 42
4
+ trackers: [jsonl, wandb]
5
+ wandb_entity: None
6
+ wandb_project: None
7
+ is_debug: false
8
+
9
+ framework:
10
+ framework_py: InternVLA-M1
11
+ qwenvl: # TODO later should not use qwenvl should use vlm directly
12
+ base_vlm: Qwen/Qwen2.5-VL-3B-Instruct
13
+ attn_implementation: flash_attention_2
14
+ vl_hidden_dim: 2048 # TODO should read from internal config
15
+ dino:
16
+ dino_backbone: dinov2_vits14
17
+ layer_qformer:
18
+ qformer_end_layer: 37
19
+ qformer_start_layer: 36
20
+ num_query_tokens: 64
21
+ input_dim: 2048 # here is VLM output dimension
22
+ ouptput_dim: 768 # here is action input dimension # here has a rule that ouptput_dim = action_hidden_dim; --> top2down logic should not default match, should be modified manually
23
+ grad_scale: 0.5 # let gradient pass through this module to decay, avoid destroying VLM, # seems to affect learning
24
+
25
+ action_model:
26
+ action_model_type: DiT-B
27
+ action_hidden_dim: 768
28
+ action_dim: 36 # 32
29
+ use_ema: false
30
+ future_action_window_size: 15 # 29
31
+ past_action_window_size: 0
32
+ repeated_diffusion_steps: 8
33
+
34
+ fm_head_config:
35
+ input_embedding_dim: 1536 # Action embedding after dimension 48 * 32,and DiT's attention_head_dim * attention_heads corresponds to ActionEncoder
36
+ hidden_size: 1024 # and DiT's last projection corresponds to ActionDecoder
37
+ add_pos_embed: True
38
+ max_seq_len: 1024
39
+ action_dim: 36 # 32
40
+ future_action_window_size: 15 # 29
41
+ action_horizon: 16 # 30 # = future_action_window_size + 1 + past_action_window_size
42
+ past_action_window_size: 0
43
+ noise_beta_alpha: 1.5
44
+ noise_beta_beta: 1.0
45
+ noise_s: 0.999
46
+ num_timestep_buckets: 1000
47
+ num_inference_timesteps: 4
48
+ num_target_vision_tokens: 32
49
+ diffusion_model_cfg: # DiT Transformers parameters
50
+ attention_head_dim: 48
51
+ cross_attention_dim: 2048 # VLM dim
52
+ dropout: 0.2
53
+ final_dropout: true
54
+ interleave_self_attention: true
55
+ norm_type: "ada_norm"
56
+ num_attention_heads: 32
57
+ num_layers: 16
58
+ output_dim: 1024
59
+ positional_embeddings: null
60
+
61
+ datasets:
62
+ vlm_data:
63
+ dataset_py: vlm_datasets
64
+ dataformat: llava_json
65
+ dataset_use: asv2_conversation_en,asv2_detailed_description_en,asv2_region_captioning_en,coco_internvl_longcap_en,coco_karpathy_train_567_en,coco_negative_gpt4o_en,coco_poetry_zh,coco_rem_en_zh,cocorem_exist_yorn_en,cocotextv2_en,cocotextv2_gpt4o_en,okvqa_en,refcoco_grounding_aug_en,refcoco_grounding_en,tallyqa_coco_en,toloka_grounding_aug_en,vqav2_en,vsr_en
66
+ eval_dataset: aokvqa_cauldron_llava_format
67
+ data_flatten: false
68
+ base_interval: 2
69
+ max_pixels: 50176
70
+ min_pixels: 784
71
+ model_max_length: 2048 # seems to be 4096 here, some related content is not processed
72
+ model_type: qwen2.5vl
73
+ per_device_batch_size: 4
74
+
75
+ vla_data:
76
+ dataset_py: lerobot_datasets
77
+ data_root_dir: None
78
+ data_mix: humanoid_
79
+ action_type: abs_joints
80
+ CoT_prompt: "Your task is {instruction}. To identify the key objects for your task. Locate their bounding boxes in [x1,y1,x2,y2] format."
81
+ CoT_answer: bbox
82
+ default_image_resolution: [3, 224, 224] # this is not effective
83
+ per_device_batch_size: 64 # FIXME
84
+ preload_all: true #FIXME
85
+ load_all_data_for_training: true # consider how to enable eval set
86
+ obs: ["image_0"] # this is not effective
87
+ image_size: [224,224] #todo is null keep raw size # this is not effective
88
+
89
+ trainer:
90
+ epochs: 100 # not effective; use max_train_steps instead
91
+ max_train_steps: 30000 #100000 #FIXME
92
+ num_warmup_steps: 0
93
+ save_interval: 5000 # 500
94
+ eval_interval: 100
95
+ learning_rate: # you can set different lr for different modules #FIXME
96
+ base: 5e-05
97
+ qwen_vl_interface: 1.0e-05
98
+ action_model: 1.0e-04
99
+ lr_scheduler_type: cosine_with_min_lr
100
+ scheduler_specific_kwargs:
101
+ min_lr: 5.0e-07
102
+ freeze_modules: 'qwen_vl_interface,layer_qformer,dino_encoder,dino_pro' # FIXME
103
+ loss_scale:
104
+ vla: 1.0
105
+ vlm: 0.1
106
+ pretrained_checkpoint: /hfm/cache/checkpoints/InternVLA-M1-Pretrain-RT-1-Bridge/checkpoints/steps_50000_pytorch_model.pt
107
+ # reload_modules: 'action_model,layer_qformer' # warning: this may cause model to not converge
108
+ skip_reload_modules: 'action_model' # skip action-related layers # FIXME
109
+ repeated_diffusion_steps: 4
110
+ max_grad_norm: 1.0
111
+ warmup_ratio: 0.1
112
+ weight_decay: 0.0
113
+ logging_frequency: 10
114
+ gradient_clipping: 1.0
115
+ gradient_accumulation_steps: 1
116
+
117
+ optimizer:
118
+ name: AdamW
119
+ betas: [0.9, 0.95]
120
+ eps: 1.0e-08
121
+ weight_decay: 1.0e-08
122
+
123
+ # parameters to be determined
124
+ is_resume: false
125
+ resume_epoch: null
126
+ resume_step: null
127
+ enable_gradient_checkpointing: true
128
+ enable_mixed_precision_training: true
src/InternVLA-M1/InternVLA/config/training/internvla_cotrain_libero.yaml ADDED
@@ -0,0 +1,126 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ run_id: internVLA
2
+ run_root_dir: results/Checkpoints
3
+ seed: 42
4
+ trackers: [jsonl, wandb]
5
+ wandb_entity: your_wandb_entity
6
+ wandb_project: internVLA
7
+ is_debug: false
8
+
9
+
10
+ framework:
11
+ framework_py: InternVLA-M1
12
+ qwenvl:
13
+ base_vlm: ./playground/Pretrained_models/Qwen2.5-VL-3B-Instruct
14
+ attn_implementation: flash_attention_2
15
+ vl_hidden_dim: 2048
16
+ dino:
17
+ dino_backbone: dinov2_vits14
18
+ layer_qformer:
19
+ qformer_end_layer: 37
20
+ qformer_start_layer: 36
21
+ num_query_tokens: 64
22
+ input_dim: 2048
23
+ ouptput_dim: 768
24
+ grad_scale: 0.5
25
+ action_model:
26
+ action_model_type: DiT-B
27
+ action_hidden_dim: 768
28
+ action_dim: 7
29
+ use_ema: false
30
+ future_action_window_size: 15
31
+ past_action_window_size: 0
32
+ repeated_diffusion_steps: 8
33
+
34
+ fm_head_config:
35
+ input_embedding_dim: 1536 # Action embedding after dimension 48 * 32,and DiT's attention_head_dim * attention_heads corresponds to ActionEncoder
36
+ hidden_size: 1024 # and DiT's last projection corresponds to ActionDecoder
37
+ add_pos_embed: True
38
+ max_seq_len: 1024
39
+ action_dim: 7
40
+ future_action_window_size: 15
41
+ action_horizon: 16
42
+ past_action_window_size: 0
43
+ noise_beta_alpha: 1.5
44
+ noise_beta_beta: 1.0
45
+ noise_s: 0.999
46
+ num_timestep_buckets: 1000
47
+ num_inference_timesteps: 4
48
+ num_target_vision_tokens: 32
49
+ diffusion_model_cfg: # DiT Transformers parameters
50
+ attention_head_dim: 48
51
+ cross_attention_dim: 2048 # VLM dim
52
+ dropout: 0.2
53
+ final_dropout: true
54
+ interleave_self_attention: true
55
+ norm_type: "ada_norm"
56
+ num_attention_heads: 32
57
+ num_layers: 16
58
+ output_dim: 1024
59
+ positional_embeddings: null
60
+
61
+ reduce_in_full_precision: true
62
+
63
+
64
+ datasets:
65
+ vlm_data:
66
+ dataset_py: vlm_datasets
67
+ dataformat: llava_json
68
+ dataset_use: asv2_conversation_en,asv2_detailed_description_en,asv2_region_captioning_en,coco_internvl_longcap_en,coco_karpathy_train_567_en,coco_negative_gpt4o_en,coco_poetry_zh,coco_rem_en_zh,cocorem_exist_yorn_en,cocotextv2_en,cocotextv2_gpt4o_en,okvqa_en,refcoco_grounding_aug_en,refcoco_grounding_en,tallyqa_coco_en,toloka_grounding_aug_en,vqav2_en,vsr_en
69
+ eval_dataset: aokvqa_cauldron_llava_format
70
+ data_flatten: false
71
+ base_interval: 2
72
+ max_pixels: 12845056
73
+ min_pixels: 3136
74
+ model_max_length: 2048
75
+ model_type: qwen2.5vl
76
+ per_device_batch_size: 4
77
+
78
+ vla_data:
79
+ dataset_py: lerobot_datasets
80
+ data_root_dir: playground/Datasets/LEROBOT_LIBERO_DATA
81
+ data_mix: libero_goal # libero_goal
82
+ action_type: delta_qpos
83
+ CoT_prompt: "Your task is {instruction}. To identify the key objects for your task. Locate their bounding boxes in [x1,y1,x2,y2] format."
84
+ CoT_answer: bbox
85
+ default_image_resolution: [3, 224, 224]
86
+ per_device_batch_size: 16
87
+ load_all_data_for_training: true
88
+ obs: ["image_0"]
89
+
90
+
91
+ trainer:
92
+ epochs: 100
93
+ max_train_steps: 100000
94
+ num_warmup_steps: 5000
95
+ save_interval: 5000
96
+ eval_interval: 100
97
+ learning_rate:
98
+ base: 2.5e-05
99
+ qwen_vl_interface: 1.0e-05
100
+ action_model: 1.0e-04
101
+ lr_scheduler_type: cosine_with_min_lr
102
+ scheduler_specific_kwargs:
103
+ min_lr: 1.0e-06
104
+ freeze_modules: ''
105
+ loss_scale:
106
+ vla: 1.0
107
+ vlm: 0.1
108
+ max_grad_norm: 1.0
109
+ warmup_ratio: 0.1
110
+ weight_decay: 0.0
111
+ logging_frequency: 10
112
+ gradient_clipping: 1.0
113
+ gradient_accumulation_steps: 1
114
+
115
+ optimizer:
116
+ name: AdamW
117
+ betas: [0.9, 0.95]
118
+ eps: 1.0e-08
119
+ weight_decay: 1.0e-08
120
+
121
+ # parameters to be determined
122
+ is_resume: false
123
+ resume_epoch: null
124
+ resume_step: null
125
+ enable_gradient_checkpointing: true
126
+ enable_mixed_precision_training: true
src/InternVLA-M1/InternVLA/config/training/internvla_cotrain_oxe.yaml ADDED
@@ -0,0 +1,100 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ run_id: qwenact
2
+ run_root_dir: results/Checkpoints
3
+ seed: 42
4
+ trackers: [jsonl, wandb]
5
+ wandb_entity: your_wandb_entity
6
+ wandb_project: internVLAAA
7
+ is_debug: false
8
+
9
+ framework:
10
+ framework_py: InternVLA-M1
11
+ qwenvl:
12
+ base_vlm: ./playground/Pretrained_models/Qwen2.5-VL-3B-Instruct
13
+ attn_implementation: flash_attention_2
14
+ vl_hidden_dim: 2048
15
+ dino:
16
+ dino_backbone: dinov2_vits14
17
+ layer_qformer:
18
+ qformer_end_layer: 37
19
+ qformer_start_layer: 36
20
+ num_query_tokens: 64
21
+ input_dim: 2048
22
+ ouptput_dim: 768
23
+ grad_scale: 0.5
24
+
25
+ action_model:
26
+ action_model_type: DiT-B
27
+ action_hidden_dim: 768
28
+ action_dim: 7
29
+ use_ema: false
30
+ future_action_window_size: 15
31
+ past_action_window_size: 0
32
+ repeated_diffusion_steps: 8
33
+
34
+
35
+ datasets:
36
+ vlm_data:
37
+ dataset_py: vlm_datasets
38
+ dataformat: llava_json
39
+ dataset_use: asv2_conversation_en,asv2_detailed_description_en,asv2_region_captioning_en,coco_internvl_longcap_en,coco_karpathy_train_567_en,coco_negative_gpt4o_en,coco_poetry_zh,coco_rem_en_zh,cocorem_exist_yorn_en,cocotextv2_en,cocotextv2_gpt4o_en,okvqa_en,refcoco_grounding_aug_en,refcoco_grounding_en,tallyqa_coco_en,toloka_grounding_aug_en,vqav2_en,vsr_en
40
+ eval_dataset: aokvqa_cauldron_llava_format
41
+ data_flatten: false
42
+ base_interval: 2
43
+ max_pixels: 50176
44
+ min_pixels: 784
45
+ model_max_length: 2048
46
+ model_type: qwen2.5vl
47
+ per_device_batch_size: 4
48
+
49
+ vla_data:
50
+ dataset_py: lerobot_datasets
51
+ data_root_dir: playground/Datasets/OXE_LEROBOT_DATASET
52
+ data_mix: bridge_rt_1 # bridge_rt_1
53
+ action_type: delta_ee
54
+ CoT_prompt: "Your task is {instruction}. To identify the key objects for your task. Locate their bounding boxes in [x1,y1,x2,y2] format."
55
+ CoT_answer: bbox
56
+ default_image_resolution: [3, 224, 224]
57
+ per_device_batch_size: 16
58
+ load_all_data_for_training: true
59
+ obs: ["image_0"]
60
+ image_size: [224,224]
61
+
62
+ trainer:
63
+ epochs: 100
64
+ max_train_steps: 100000
65
+ num_warmup_steps: 5000
66
+ save_interval: 5000
67
+ eval_interval: 100
68
+ learning_rate:
69
+ base: 1e-05
70
+ qwen_vl_interface: 1.0e-05
71
+ action_model: 1.0e-04
72
+ lr_scheduler_type: cosine_with_min_lr
73
+ scheduler_specific_kwargs:
74
+ min_lr: 5.0e-07
75
+ freeze_modules: ''
76
+ loss_scale:
77
+ vla: 1.0
78
+ vlm: 0.1
79
+ # pretrained_checkpoint: ./results/Checkpoints/1_need/0601_qwenact_fixqwen_32gpus_lr_1e-3_qformer_36_37/steps_100000/pytorch_model.pt
80
+ # reload_modules: 'action_model,layer_qformer' # warning: this may cause model to not converge
81
+ repeated_diffusion_steps: 4
82
+ max_grad_norm: 1.0
83
+ warmup_ratio: 0.1
84
+ weight_decay: 0.0
85
+ logging_frequency: 10
86
+ gradient_clipping: 1.0
87
+ gradient_accumulation_steps: 1
88
+
89
+ optimizer:
90
+ name: AdamW
91
+ betas: [0.9, 0.95]
92
+ eps: 1.0e-08
93
+ weight_decay: 1.0e-08
94
+
95
+ # parameters to be determined
96
+ is_resume: false
97
+ resume_epoch: null
98
+ resume_step: null
99
+ enable_gradient_checkpointing: true
100
+ enable_mixed_precision_training: true
src/InternVLA-M1/InternVLA/config/training/internvla_cotrain_sim_demo.yaml ADDED
@@ -0,0 +1,124 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ run_id: qwenact
2
+ run_root_dir: results/Checkpoints
3
+ seed: 42
4
+ trackers: [jsonl, wandb]
5
+ wandb_entity: ailab-manipulation
6
+ wandb_project: internVLA
7
+ is_debug: false
8
+
9
+ framework:
10
+ framework_py: InternVLA-M1
11
+ qwenvl: # TODO later should not use qwenvl should use vlm directly
12
+ base_vlm: ./playground/Pretrained_models/Qwen2.5-VL-3B-Instruct
13
+ attn_implementation: flash_attention_2
14
+ vl_hidden_dim: 2048 # TODO should read from internal config
15
+ dino:
16
+ dino_backbone: dinov2_vits14
17
+ layer_qformer:
18
+ qformer_end_layer: 37
19
+ qformer_start_layer: 36
20
+ num_query_tokens: 64
21
+ input_dim: 2048 # here is VLM output dimension
22
+ ouptput_dim: 768 # here is action input dimension # here has a rule that ouptput_dim = action_hidden_dim; --> top2down logic should not default match, should be modified manually
23
+ grad_scale: 0.5 # let gradient pass through this module to decay, avoid destroying VLM, # seems to affect learning
24
+
25
+ action_model:
26
+ action_model_type: DiT-B
27
+ action_hidden_dim: 768
28
+ action_dim: 8
29
+ use_ema: false
30
+ future_action_window_size: 15
31
+ past_action_window_size: 0
32
+ repeated_diffusion_steps: 8
33
+
34
+ fm_head_config:
35
+ input_embedding_dim: 1536 # Action embedding after dimension 48 * 32,and DiT's attention_head_dim * attention_heads corresponds to ActionEncoder
36
+ hidden_size: 1024 # and DiT's last projection corresponds to ActionDecoder
37
+ add_pos_embed: True
38
+ max_seq_len: 1024
39
+ action_dim: 8
40
+ future_action_window_size: 15
41
+ action_horizon: 16 # = future_action_window_size + 1 + past_action_window_size
42
+ past_action_window_size: 0
43
+ noise_beta_alpha: 1.5
44
+ noise_beta_beta: 1.0
45
+ noise_s: 0.999
46
+ num_timestep_buckets: 1000
47
+ num_inference_timesteps: 4
48
+ num_target_vision_tokens: 32
49
+ diffusion_model_cfg: # DiT Transformers parameters
50
+ attention_head_dim: 48
51
+ cross_attention_dim: 2048 # VLM dim
52
+ dropout: 0.2
53
+ final_dropout: true
54
+ interleave_self_attention: true
55
+ norm_type: "ada_norm"
56
+ num_attention_heads: 32
57
+ num_layers: 16
58
+ output_dim: 1024
59
+ positional_embeddings: null
60
+
61
+ datasets:
62
+ vlm_data:
63
+ dataset_py: vlm_datasets
64
+ dataformat: llava_json
65
+ dataset_use: asv2_conversation_en,asv2_detailed_description_en,asv2_region_captioning_en,coco_internvl_longcap_en,coco_karpathy_train_567_en,coco_negative_gpt4o_en,coco_poetry_zh,coco_rem_en_zh,cocorem_exist_yorn_en,cocotextv2_en,cocotextv2_gpt4o_en,okvqa_en,refcoco_grounding_aug_en,refcoco_grounding_en,tallyqa_coco_en,toloka_grounding_aug_en,vqav2_en,vsr_en
66
+ eval_dataset: aokvqa_cauldron_llava_format
67
+ data_flatten: false
68
+ base_interval: 2
69
+ max_pixels: 50176
70
+ min_pixels: 784
71
+ model_max_length: 2048 # seems to be 4096 here, some related content is not processed
72
+ model_type: qwen2.5vl
73
+ per_device_batch_size: 4
74
+
75
+ vla_data:
76
+ dataset_py: lerobot_datasets
77
+ data_root_dir: playground/demo_data
78
+ data_mix: demo_sim_pick_place
79
+ action_type: delta_joints
80
+ CoT_prompt: "Your task is {instruction}. To identify the key objects for your task. Locate their bounding boxes in [x1,y1,x2,y2] format."
81
+ CoT_answer: bbox
82
+ default_image_resolution: [3, 224, 224] # this is not effective
83
+ per_device_batch_size: 16
84
+ load_all_data_for_training: true # consider how to enable eval set
85
+ obs: ["image_0"] # this is not effective
86
+ image_size: [224,224] #todo is null keep raw size # this is not effective
87
+
88
+ trainer:
89
+ epochs: 100 # decide whether to use, he and max_train_steps conflict
90
+ max_train_steps: 100000
91
+ num_warmup_steps: 5000
92
+ save_interval: 5000
93
+ eval_interval: 100
94
+ learning_rate: # you can set different lr for different modules
95
+ base: 5e-05
96
+ qwen_vl_interface: 1.0e-05
97
+ action_model: 1.0e-04
98
+ lr_scheduler_type: cosine_with_min_lr
99
+ scheduler_specific_kwargs:
100
+ min_lr: 5.0e-07
101
+ freeze_modules: ''
102
+ loss_scale:
103
+ vla: 1.0
104
+ vlm: 0.1
105
+ repeated_diffusion_steps: 4
106
+ max_grad_norm: 1.0
107
+ warmup_ratio: 0.1
108
+ weight_decay: 0.0
109
+ logging_frequency: 10
110
+ gradient_clipping: 1.0
111
+ gradient_accumulation_steps: 1
112
+
113
+ optimizer:
114
+ name: AdamW
115
+ betas: [0.9, 0.95]
116
+ eps: 1.0e-08
117
+ weight_decay: 1.0e-08
118
+
119
+ # parameters to be determined
120
+ is_resume: false
121
+ resume_epoch: null
122
+ resume_step: null
123
+ enable_gradient_checkpointing: true
124
+ enable_mixed_precision_training: true
src/InternVLA-M1/InternVLA/dataloader/__init__.py ADDED
@@ -0,0 +1,63 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import json
2
+ import os
3
+ from accelerate.logging import get_logger
4
+ import numpy as np
5
+ from torch.utils.data import DataLoader
6
+ import numpy as np
7
+ import torch.distributed as dist
8
+ from pathlib import Path
9
+ from InternVLA.dataloader.vlm_datasets import make_vlm_dataloader
10
+
11
+ logger = get_logger(__name__)
12
+
13
+ def save_dataset_statistics(dataset_statistics, run_dir):
14
+ """Saves a `dataset_statistics.json` file."""
15
+ out_path = run_dir / "dataset_statistics.json"
16
+ with open(out_path, "w") as f_json:
17
+ for _, stats in dataset_statistics.items():
18
+ for k in stats["action"].keys():
19
+ if isinstance(stats["action"][k], np.ndarray):
20
+ stats["action"][k] = stats["action"][k].tolist()
21
+ if "proprio" in stats:
22
+ for k in stats["proprio"].keys():
23
+ if isinstance(stats["proprio"][k], np.ndarray):
24
+ stats["proprio"][k] = stats["proprio"][k].tolist()
25
+ if "num_trajectories" in stats:
26
+ if isinstance(stats["num_trajectories"], np.ndarray):
27
+ stats["num_trajectories"] = stats["num_trajectories"].item()
28
+ if "num_transitions" in stats:
29
+ if isinstance(stats["num_transitions"], np.ndarray):
30
+ stats["num_transitions"] = stats["num_transitions"].item()
31
+ json.dump(dataset_statistics, f_json, indent=2)
32
+ logger.info(f"Saved dataset statistics file at path {out_path}")
33
+
34
+
35
+
36
+ def build_dataloader(cfg, dataset_py="lerobot_datasets_oxe"): # TODO now here only is get dataset, we need mv dataloader to here
37
+
38
+ if dataset_py == "lerobot_datasets":
39
+ from InternVLA.dataloader.lerobot_datasets import get_vla_dataset, collate_fn
40
+ vla_dataset_cfg = cfg.datasets.vla_data
41
+
42
+ data_root_dir = vla_dataset_cfg.data_root_dir
43
+ data_mix = vla_dataset_cfg.data_mix
44
+
45
+ vla_dataset = get_vla_dataset(data_cfg=vla_dataset_cfg)
46
+
47
+ vla_train_dataloader = DataLoader(
48
+ vla_dataset,
49
+ batch_size=cfg.datasets.vla_data.per_device_batch_size,
50
+ collate_fn=collate_fn,
51
+ num_workers=8,
52
+ # shuffle=True
53
+ )
54
+ if dist.get_rank() == 0:
55
+
56
+ output_dir = Path(cfg.output_dir)
57
+ vla_dataset.save_dataset_statistics(output_dir / "dataset_statistics.json")
58
+ return vla_train_dataloader
59
+ elif dataset_py == "vlm_datasets":
60
+ vlm_data_module = make_vlm_dataloader(cfg)
61
+ vlm_train_dataloader = vlm_data_module["train_dataloader"]
62
+
63
+ return vlm_train_dataloader
src/InternVLA-M1/InternVLA/dataloader/gr00t_lerobot/README.md ADDED
File without changes
src/InternVLA-M1/InternVLA/dataloader/gr00t_lerobot/__init__.py ADDED
File without changes
src/InternVLA-M1/InternVLA/dataloader/gr00t_lerobot/data_config.py ADDED
@@ -0,0 +1,770 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
2
+ # SPDX-License-Identifier: Apache-2.0
3
+ #
4
+ # Licensed under the Apache License, Version 2.0 (the "License");
5
+ # you may not use this file except in compliance with the License.
6
+ # You may obtain a copy of the License at
7
+ #
8
+ # http://www.apache.org/licenses/LICENSE-2.0
9
+ #
10
+ # Unless required by applicable law or agreed to in writing, software
11
+ # distributed under the License is distributed on an "AS IS" BASIS,
12
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
+ # See the License for the specific language governing permissions and
14
+ # limitations under the License.
15
+
16
+ from abc import ABC, abstractmethod
17
+
18
+ from InternVLA.dataloader.gr00t_lerobot.datasets import ModalityConfig
19
+ from InternVLA.dataloader.gr00t_lerobot.transform.base import ComposedModalityTransform, ModalityTransform
20
+ from InternVLA.dataloader.gr00t_lerobot.transform.concat import ConcatTransform
21
+ from InternVLA.dataloader.gr00t_lerobot.transform.state_action import (
22
+ StateActionSinCosTransform,
23
+ StateActionToTensor,
24
+ StateActionTransform,
25
+ )
26
+ from InternVLA.dataloader.gr00t_lerobot.transform.video import (
27
+ VideoColorJitter,
28
+ VideoCrop,
29
+ VideoResize,
30
+ VideoToNumpy,
31
+ VideoToTensor,
32
+ )
33
+ # from gr00t.model.transforms import GR00TTransform
34
+
35
+
36
+ class BaseDataConfig(ABC):
37
+ @abstractmethod
38
+ def modality_config(self) -> dict[str, ModalityConfig]:
39
+ pass
40
+
41
+ @abstractmethod
42
+ def transform(self) -> ModalityTransform:
43
+ pass
44
+
45
+
46
+ ###########################################################################################
47
+
48
+ class OxeDroidDataConfig:
49
+ video_keys = [
50
+ "video.exterior_image_1",
51
+ "video.exterior_image_2",
52
+ "video.wrist_image",
53
+ ]
54
+ state_keys = [
55
+ "state.eef_position",
56
+ "state.eef_rotation",
57
+ "state.gripper_position",
58
+ ]
59
+ action_keys = [
60
+ "action.eef_position_delta",
61
+ "action.eef_rotation_delta",
62
+ "action.gripper_position",
63
+ ]
64
+ language_keys = ["annotation.language.language_instruction"]
65
+ observation_indices = [0]
66
+ action_indices = list(range(16))
67
+
68
+ def modality_config(self):
69
+ video_modality = ModalityConfig(
70
+ delta_indices=self.observation_indices,
71
+ modality_keys=self.video_keys,
72
+ )
73
+ state_modality = ModalityConfig(
74
+ delta_indices=self.observation_indices,
75
+ modality_keys=self.state_keys,
76
+ )
77
+ action_modality = ModalityConfig(
78
+ delta_indices=self.action_indices,
79
+ modality_keys=self.action_keys,
80
+ )
81
+ language_modality = ModalityConfig(
82
+ delta_indices=self.observation_indices,
83
+ modality_keys=self.language_keys,
84
+ )
85
+ modality_configs = {
86
+ "video": video_modality,
87
+ "state": state_modality,
88
+ "action": action_modality,
89
+ "language": language_modality,
90
+ }
91
+ return modality_configs
92
+
93
+ def transform(self):
94
+ transforms = [
95
+ # video transforms
96
+ VideoToTensor(apply_to=self.video_keys),
97
+ VideoCrop(apply_to=self.video_keys, scale=0.95),
98
+ VideoResize(apply_to=self.video_keys, height=224, width=224, interpolation="linear"),
99
+ VideoColorJitter(
100
+ apply_to=self.video_keys,
101
+ brightness=0.3,
102
+ contrast=0.4,
103
+ saturation=0.5,
104
+ hue=0.08,
105
+ ),
106
+ VideoToNumpy(apply_to=self.video_keys),
107
+ # state transforms
108
+ StateActionToTensor(apply_to=self.state_keys),
109
+ StateActionTransform(
110
+ apply_to=self.state_keys,
111
+ normalization_modes={
112
+ "state.eef_position": "min_max",
113
+ "state.gripper_position": "min_max",
114
+ },
115
+ target_rotations={
116
+ "state.eef_rotation": "rotation_6d",
117
+ },
118
+ ),
119
+ # action transforms
120
+ StateActionToTensor(apply_to=self.action_keys),
121
+ StateActionTransform(
122
+ apply_to=self.action_keys,
123
+ normalization_modes={
124
+ "action.gripper_position": "binary",
125
+ },
126
+ target_rotations={"action.eef_rotation_delta": "axis_angle"},
127
+ ),
128
+ # concat transforms
129
+ ConcatTransform(
130
+ video_concat_order=self.video_keys,
131
+ state_concat_order=self.state_keys,
132
+ action_concat_order=self.action_keys,
133
+ ),
134
+ GR00TTransform(
135
+ state_horizon=len(self.observation_indices),
136
+ action_horizon=len(self.action_indices),
137
+ max_state_dim=64,
138
+ max_action_dim=32,
139
+ ),
140
+ ]
141
+
142
+ return ComposedModalityTransform(transforms=transforms)
143
+
144
+
145
+ ###########################################################################################
146
+
147
+
148
+ class OxeBridgeDataConfig:
149
+ video_keys = [
150
+ "video.image_0",
151
+ ]
152
+ state_keys = [
153
+ "state.x",
154
+ "state.y",
155
+ "state.z",
156
+ "state.roll",
157
+ "state.pitch",
158
+ "state.yaw",
159
+ "state.pad",
160
+ "state.gripper",
161
+ ]
162
+ action_keys = [
163
+ "action.x",
164
+ "action.y",
165
+ "action.z",
166
+ "action.roll",
167
+ "action.pitch",
168
+ "action.yaw",
169
+ "action.gripper",
170
+ ]
171
+ language_keys = ["annotation.human.action.task_description"]
172
+ observation_indices = [0]
173
+ action_indices = list(range(16))
174
+
175
+ def modality_config(self):
176
+ video_modality = ModalityConfig(
177
+ delta_indices=self.observation_indices,
178
+ modality_keys=self.video_keys,
179
+ )
180
+ state_modality = ModalityConfig(
181
+ delta_indices=self.observation_indices,
182
+ modality_keys=self.state_keys,
183
+ )
184
+ action_modality = ModalityConfig(
185
+ delta_indices=self.action_indices,
186
+ modality_keys=self.action_keys,
187
+ )
188
+ language_modality = ModalityConfig(
189
+ delta_indices=self.observation_indices,
190
+ modality_keys=self.language_keys,
191
+ )
192
+ modality_configs = {
193
+ "video": video_modality,
194
+ "state": state_modality,
195
+ "action": action_modality,
196
+ "language": language_modality,
197
+ }
198
+ return modality_configs
199
+
200
+ def transform(self):
201
+ transforms = [
202
+ # video transforms
203
+ # VideoToTensor(apply_to=self.video_keys),
204
+ # VideoCrop(apply_to=self.video_keys, scale=0.95),
205
+ # VideoResize(apply_to=self.video_keys, height=224, width=224, interpolation="linear"),
206
+ # VideoColorJitter(
207
+ # apply_to=self.video_keys,
208
+ # brightness=0.3,
209
+ # contrast=0.4,
210
+ # saturation=0.5,
211
+ # hue=0.08,
212
+ # ),
213
+ # VideoToNumpy(apply_to=self.video_keys),
214
+ # state transforms
215
+ StateActionToTensor(apply_to=self.state_keys),
216
+ StateActionTransform(
217
+ apply_to=self.state_keys,
218
+ normalization_modes={
219
+ "state.x": "q99",
220
+ "state.y": "q99",
221
+ "state.z": "q99",
222
+ "state.roll": "q99",
223
+ "state.pitch": "q99",
224
+ "state.yaw": "q99",
225
+ "state.pad": "q99",
226
+ "state.gripper": "binary",
227
+ },
228
+ ),
229
+ # action transforms
230
+ StateActionToTensor(apply_to=self.action_keys),
231
+ StateActionTransform(
232
+ apply_to=self.action_keys,
233
+ normalization_modes={
234
+ "action.x": "q99",
235
+ "action.y": "q99",
236
+ "action.z": "q99",
237
+ "action.roll": "q99",
238
+ "action.pitch": "q99",
239
+ "action.yaw": "q99",
240
+ "action.gripper": "binary",
241
+ },
242
+ ),
243
+ # concat transforms
244
+ # ConcatTransform(
245
+ # # video_concat_order=self.video_keys,
246
+ # state_concat_order=self.state_keys,
247
+ # action_concat_order=self.action_keys,
248
+ # ),
249
+ # GR00TTransform(
250
+ # state_horizon=len(self.observation_indices),
251
+ # action_horizon=len(self.action_indices),
252
+ # max_state_dim=64,
253
+ # max_action_dim=32,
254
+ # ),
255
+ ]
256
+
257
+ return ComposedModalityTransform(transforms=transforms)
258
+
259
+
260
+ ###########################################################################################
261
+
262
+ class OxeRT1DataConfig:
263
+ video_keys = [
264
+ "video.image",
265
+ ]
266
+ state_keys = [
267
+ "state.x",
268
+ "state.y",
269
+ "state.z",
270
+ "state.rx",
271
+ "state.ry",
272
+ "state.rz",
273
+ "state.rw",
274
+ "state.gripper",
275
+ ]
276
+ action_keys = [
277
+ "action.x",
278
+ "action.y",
279
+ "action.z",
280
+ "action.roll",
281
+ "action.pitch",
282
+ "action.yaw",
283
+ "action.gripper",
284
+ ]
285
+ language_keys = ["annotation.human.action.task_description"]
286
+ observation_indices = [0]
287
+ action_indices = list(range(16))
288
+
289
+ def modality_config(self):
290
+ video_modality = ModalityConfig(
291
+ delta_indices=self.observation_indices,
292
+ modality_keys=self.video_keys,
293
+ )
294
+ state_modality = ModalityConfig(
295
+ delta_indices=self.observation_indices,
296
+ modality_keys=self.state_keys,
297
+ )
298
+ action_modality = ModalityConfig(
299
+ delta_indices=self.action_indices,
300
+ modality_keys=self.action_keys,
301
+ )
302
+ language_modality = ModalityConfig(
303
+ delta_indices=self.observation_indices,
304
+ modality_keys=self.language_keys,
305
+ )
306
+ modality_configs = {
307
+ "video": video_modality,
308
+ "state": state_modality,
309
+ "action": action_modality,
310
+ "language": language_modality,
311
+ }
312
+ return modality_configs
313
+
314
+ def transform(self):
315
+ transforms = [
316
+ # video transforms
317
+ # VideoToTensor(apply_to=self.video_keys),
318
+ # VideoCrop(apply_to=self.video_keys, scale=0.95),
319
+ # VideoResize(apply_to=self.video_keys, height=224, width=224, interpolation="linear"),
320
+ # VideoColorJitter(
321
+ # apply_to=self.video_keys,
322
+ # brightness=0.3,
323
+ # contrast=0.4,
324
+ # saturation=0.5,
325
+ # hue=0.08,
326
+ # ),
327
+ # VideoToNumpy(apply_to=self.video_keys),
328
+ # state transforms
329
+ StateActionToTensor(apply_to=self.state_keys),
330
+ StateActionTransform(
331
+ apply_to=self.state_keys,
332
+ normalization_modes={
333
+ "state.x": "q99",
334
+ "state.y": "q99",
335
+ "state.z": "q99",
336
+ "state.rx": "q99",
337
+ "state.ry": "q99",
338
+ "state.rz": "q99",
339
+ "state.rw": "q99",
340
+ "state.gripper": "binary",
341
+ },
342
+ ),
343
+ # action transforms
344
+ StateActionToTensor(apply_to=self.action_keys),
345
+ StateActionTransform(
346
+ apply_to=self.action_keys,
347
+ normalization_modes={
348
+ "action.x": "q99",
349
+ "action.y": "q99",
350
+ "action.z": "q99",
351
+ "action.roll": "q99",
352
+ "action.pitch": "q99",
353
+ "action.yaw": "q99",
354
+ "action.gripper": "binary",
355
+ },
356
+ ),
357
+ # concat transforms
358
+ # ConcatTransform(
359
+ # # video_concat_order=self.video_keys,
360
+ # state_concat_order=self.state_keys,
361
+ # action_concat_order=self.action_keys,
362
+ # ),
363
+ # GR00TTransform(
364
+ # state_horizon=len(self.observation_indices),
365
+ # action_horizon=len(self.action_indices),
366
+ # max_state_dim=64,
367
+ # max_action_dim=32,
368
+ # ),
369
+ ]
370
+
371
+ return ComposedModalityTransform(transforms=transforms)
372
+
373
+
374
+ ###########################################################################################
375
+
376
+
377
+ class SingleFrankaRobotiqDeltaEefDataConfig:
378
+ video_keys = [
379
+ "video.base_view",
380
+ "video.ego_view",
381
+ ]
382
+ state_keys = [
383
+ "state.eef_position",
384
+ "state.eef_rotation",
385
+ ]
386
+ action_keys = [
387
+ "action.delta_eef_position",
388
+ "action.delta_eef_rotation",
389
+ "action.gripper_close",
390
+ ]
391
+
392
+ language_keys = ["annotation.human.action.task_description"]
393
+ observation_indices = [0]
394
+ action_indices = list(range(16))
395
+
396
+ def modality_config(self):
397
+ video_modality = ModalityConfig(
398
+ delta_indices=self.observation_indices,
399
+ modality_keys=self.video_keys,
400
+ )
401
+ state_modality = ModalityConfig(
402
+ delta_indices=self.observation_indices,
403
+ modality_keys=self.state_keys,
404
+ )
405
+ action_modality = ModalityConfig(
406
+ delta_indices=self.action_indices,
407
+ modality_keys=self.action_keys,
408
+ )
409
+ language_modality = ModalityConfig(
410
+ delta_indices=self.observation_indices,
411
+ modality_keys=self.language_keys,
412
+ )
413
+ modality_configs = {
414
+ "video": video_modality,
415
+ "state": state_modality,
416
+ "action": action_modality,
417
+ "language": language_modality,
418
+ }
419
+ return modality_configs
420
+
421
+ def transform(self):
422
+ transforms = [
423
+ # state transforms
424
+ StateActionToTensor(apply_to=self.state_keys),
425
+ StateActionTransform(
426
+ apply_to=self.state_keys,
427
+ normalization_modes={
428
+ "state.eef_position": "min_max",
429
+ "state.eef_rotation": "min_max",
430
+ },
431
+ ),
432
+ # action transforms
433
+ StateActionToTensor(apply_to=self.action_keys),
434
+ StateActionTransform(
435
+ apply_to=self.action_keys,
436
+ normalization_modes={
437
+ "action.delta_eef_position": "min_max",
438
+ "action.delta_eef_rotation": "min_max",
439
+ "action.gripper_close": "binary",
440
+ },
441
+ ),
442
+ ]
443
+
444
+ return ComposedModalityTransform(transforms=transforms)
445
+
446
+ ###########################################################################################
447
+
448
+ class Libero4in1DataConfig:
449
+ video_keys = [
450
+ "video.primary_image",
451
+ "video.wrist_image",
452
+ ]
453
+
454
+ state_keys = [
455
+ "state.x",
456
+ "state.y",
457
+ "state.z",
458
+ "state.roll",
459
+ "state.pitch",
460
+ "state.yaw",
461
+ "state.pad",
462
+ "state.gripper",
463
+ ]
464
+ action_keys = [
465
+ "action.x",
466
+ "action.y",
467
+ "action.z",
468
+ "action.roll",
469
+ "action.pitch",
470
+ "action.yaw",
471
+ "action.gripper",
472
+ ]
473
+
474
+ language_keys = ["annotation.human.action.task_description"]
475
+
476
+ observation_indices = [0]
477
+ action_indices = list(range(8))
478
+
479
+
480
+ def modality_config(self):
481
+ video_modality = ModalityConfig(
482
+ delta_indices=self.observation_indices,
483
+ modality_keys=self.video_keys,
484
+ )
485
+ state_modality = ModalityConfig(
486
+ delta_indices=self.observation_indices,
487
+ modality_keys=self.state_keys,
488
+ )
489
+ action_modality = ModalityConfig(
490
+ delta_indices=self.action_indices,
491
+ modality_keys=self.action_keys,
492
+ )
493
+ language_modality = ModalityConfig(
494
+ delta_indices=self.observation_indices,
495
+ modality_keys=self.language_keys,
496
+ )
497
+ modality_configs = {
498
+ "video": video_modality,
499
+ "state": state_modality,
500
+ "action": action_modality,
501
+ "language": language_modality,
502
+ }
503
+ return modality_configs
504
+
505
+ def transform(self):
506
+ transforms = [
507
+ # action transforms
508
+ StateActionToTensor(apply_to=self.action_keys),
509
+ StateActionTransform(
510
+ apply_to=self.action_keys,
511
+ normalization_modes={
512
+ "action.x": "min_max",
513
+ "action.y": "min_max",
514
+ "action.z": "min_max",
515
+ "action.roll": "min_max",
516
+ "action.pitch": "min_max",
517
+ "action.yaw": "min_max",
518
+ },
519
+ ),
520
+ ]
521
+
522
+ return ComposedModalityTransform(transforms=transforms)
523
+
524
+ ###########################################################################################
525
+
526
+
527
+ class SingleFrankaRobotiqDeltaJointsDataConfig:
528
+ video_keys = [
529
+ "video.base_view",
530
+ "video.ego_view",
531
+ ]
532
+ state_keys = [
533
+ "state.joints",
534
+ ]
535
+ action_keys = [
536
+ "action.delta_joints",
537
+ "action.gripper_close",
538
+ ]
539
+
540
+ language_keys = ["annotation.human.action.task_description"]
541
+ observation_indices = [0]
542
+ action_indices = list(range(16))
543
+
544
+ def modality_config(self):
545
+ video_modality = ModalityConfig(
546
+ delta_indices=self.observation_indices,
547
+ modality_keys=self.video_keys,
548
+ )
549
+ state_modality = ModalityConfig(
550
+ delta_indices=self.observation_indices,
551
+ modality_keys=self.state_keys,
552
+ )
553
+ action_modality = ModalityConfig(
554
+ delta_indices=self.action_indices,
555
+ modality_keys=self.action_keys,
556
+ )
557
+ language_modality = ModalityConfig(
558
+ delta_indices=self.observation_indices,
559
+ modality_keys=self.language_keys,
560
+ )
561
+ modality_configs = {
562
+ "video": video_modality,
563
+ "state": state_modality,
564
+ "action": action_modality,
565
+ "language": language_modality,
566
+ }
567
+ return modality_configs
568
+
569
+ def transform(self):
570
+ transforms = [
571
+ # state transforms
572
+ StateActionToTensor(apply_to=self.state_keys),
573
+ StateActionTransform(
574
+ apply_to=self.state_keys,
575
+ normalization_modes={
576
+ "state.joints": "min_max",
577
+ },
578
+ ),
579
+ # action transforms
580
+ StateActionToTensor(apply_to=self.action_keys),
581
+ StateActionTransform(
582
+ apply_to=self.action_keys,
583
+ normalization_modes={
584
+ "action.delta_joints": "min_max",
585
+ "action.gripper_close": "binary",
586
+ },
587
+ ),
588
+ ]
589
+
590
+ return ComposedModalityTransform(transforms=transforms)
591
+
592
+ class HumanoidDataConfig:
593
+ video_keys = [
594
+ "video.rs_view",
595
+ ]
596
+ state_keys = [
597
+ "state.left_hand",
598
+ "state.right_hand",
599
+ "state.left_arm",
600
+ "state.right_arm",
601
+ "state.rpy",
602
+ "state.height",
603
+ ]
604
+ action_keys = [
605
+ "action.left_hand",
606
+ "action.right_hand",
607
+ "action.left_arm",
608
+ "action.right_arm",
609
+ "action.rpy",
610
+ "action.height",
611
+ "action.torso_vx",
612
+ "action.torso_vy",
613
+ "action.torso_vyaw",
614
+ "action.target_yaw",
615
+ ]
616
+
617
+ language_keys = ["annotation.human.task_description"]
618
+ observation_indices = [0]
619
+ action_indices = list(range(16)) # list(range(30))
620
+
621
+ def modality_config(self):
622
+ video_modality = ModalityConfig(
623
+ delta_indices=self.observation_indices,
624
+ modality_keys=self.video_keys,
625
+ )
626
+ state_modality = ModalityConfig(
627
+ delta_indices=self.observation_indices,
628
+ modality_keys=self.state_keys,
629
+ )
630
+ action_modality = ModalityConfig(
631
+ delta_indices=self.action_indices,
632
+ modality_keys=self.action_keys,
633
+ )
634
+ language_modality = ModalityConfig(
635
+ delta_indices=self.observation_indices,
636
+ modality_keys=self.language_keys,
637
+ )
638
+ modality_configs = {
639
+ "video": video_modality,
640
+ "state": state_modality,
641
+ "action": action_modality,
642
+ "language": language_modality,
643
+ }
644
+ return modality_configs
645
+
646
+ def transform(self):
647
+ transforms = [
648
+ # state transforms
649
+ StateActionToTensor(apply_to=self.state_keys),
650
+ StateActionTransform(
651
+ apply_to=self.state_keys,
652
+ normalization_modes={
653
+ "state.left_hand": "min_max",
654
+ "state.right_hand": "min_max",
655
+ "state.left_arm": "min_max",
656
+ "state.right_arm": "min_max",
657
+ "state.rpy": "min_max",
658
+ "state.height": "min_max",
659
+ },
660
+ ),
661
+ # action transforms
662
+ StateActionToTensor(apply_to=self.action_keys),
663
+ StateActionTransform(
664
+ apply_to=self.action_keys,
665
+ normalization_modes={
666
+ "action.left_hand": "min_max",
667
+ "action.right_hand": "min_max",
668
+ "action.left_arm": "min_max",
669
+ "action.right_arm": "min_max",
670
+ "action.rpy": "min_max",
671
+ "action.height": "min_max",
672
+ "action.torso_vx": "min_max",
673
+ "action.torso_vy": "min_max",
674
+ "action.torso_vyaw": "min_max",
675
+ "action.target_yaw": "min_max",
676
+ },
677
+ ),
678
+ ]
679
+
680
+ return ComposedModalityTransform(transforms=transforms)
681
+
682
+
683
+ class HumanoidSimDataConfig:
684
+ video_keys = ["video.rs_view"]
685
+ state_keys = [
686
+ "state.left_hand",
687
+ "state.right_hand",
688
+ "state.left_arm",
689
+ "state.right_arm",
690
+ "state.waist",
691
+ ]
692
+ action_keys = [
693
+ "action.left_hand",
694
+ "action.right_hand",
695
+ "action.left_arm",
696
+ "action.right_arm",
697
+ "action.waist",
698
+ ]
699
+ language_keys = ["annotation.human.task_description"]
700
+ observation_indices = [0]
701
+ action_indices = list(range(16))
702
+
703
+ def modality_config(self):
704
+ video_modality = ModalityConfig(
705
+ delta_indices=self.observation_indices,
706
+ modality_keys=self.video_keys,
707
+ )
708
+ state_modality = ModalityConfig(
709
+ delta_indices=self.observation_indices,
710
+ modality_keys=self.state_keys,
711
+ )
712
+ action_modality = ModalityConfig(
713
+ delta_indices=self.action_indices,
714
+ modality_keys=self.action_keys,
715
+ )
716
+ language_modality = ModalityConfig(
717
+ delta_indices=self.observation_indices,
718
+ modality_keys=self.language_keys,
719
+ )
720
+ return {
721
+ "video": video_modality,
722
+ "state": state_modality,
723
+ "action": action_modality,
724
+ "language": language_modality,
725
+ }
726
+
727
+ def transform(self):
728
+ transforms = [
729
+ VideoToTensor(apply_to=self.video_keys),
730
+ VideoCrop(apply_to=self.video_keys, scale=0.95),
731
+ VideoResize(apply_to=self.video_keys, height=224, width=224, interpolation="linear"),
732
+ VideoColorJitter(
733
+ apply_to=self.video_keys,
734
+ brightness=0.3,
735
+ contrast=0.4,
736
+ saturation=0.5,
737
+ hue=0.08,
738
+ ),
739
+ VideoToNumpy(apply_to=self.video_keys),
740
+ StateActionToTensor(apply_to=self.state_keys),
741
+ StateActionTransform(
742
+ apply_to=self.state_keys,
743
+ normalization_modes={key: "min_max" for key in self.state_keys},
744
+ ),
745
+ StateActionToTensor(apply_to=self.action_keys),
746
+ StateActionTransform(
747
+ apply_to=self.action_keys,
748
+ normalization_modes={key: "min_max" for key in self.action_keys},
749
+ ),
750
+ ConcatTransform(
751
+ video_concat_order=self.video_keys,
752
+ state_concat_order=self.state_keys,
753
+ action_concat_order=self.action_keys,
754
+ ),
755
+ ]
756
+ return ComposedModalityTransform(transforms=transforms)
757
+ ###########################################################################################
758
+
759
+
760
+
761
+ ROBOT_TYPE_CONFIG_MAP = {
762
+ "libero_franka": Libero4in1DataConfig(),
763
+ "oxe_droid": OxeDroidDataConfig(),
764
+ "oxe_bridge": OxeBridgeDataConfig(),
765
+ "oxe_rt1": OxeRT1DataConfig(),
766
+ "demo_sim_franka_delta_joints": SingleFrankaRobotiqDeltaJointsDataConfig(),
767
+ "custom_robot_config": SingleFrankaRobotiqDeltaEefDataConfig(),
768
+ "humanoid_config": HumanoidDataConfig(),
769
+ "humanoid_sim_config": HumanoidSimDataConfig(),
770
+ }
src/InternVLA-M1/InternVLA/dataloader/gr00t_lerobot/datasets.py ADDED
@@ -0,0 +1,2118 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
2
+ # SPDX-License-Identifier: Apache-2.0
3
+ #
4
+ # Licensed under the Apache License, Version 2.0 (the "License");
5
+ # you may not use this file except in compliance with the License.
6
+ # You may obtain a copy of the License at
7
+ #
8
+ # http://www.apache.org/licenses/LICENSE-2.0
9
+ #
10
+ # Unless required by applicable law or agreed to in writing, software
11
+ # distributed under the License is distributed on an "AS IS" BASIS,
12
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
+ # See the License for the specific language governing permissions and
14
+ # limitations under the License.
15
+
16
+
17
+ """
18
+ In this file, we define 3 types of datasets:
19
+ 1. LeRobotSingleDataset: a single dataset for a given embodiment tag
20
+ 2. LeRobotMixtureDataset: a mixture of datasets for a given list of embodiment tags
21
+ 3. CachedLeRobotSingleDataset: a single dataset for a given embodiment tag,
22
+ with caching for the video frames
23
+
24
+ See `scripts/load_dataset.py` for examples on how to use these datasets.
25
+ """
26
+
27
+ import hashlib
28
+ import json
29
+ from collections import defaultdict
30
+ from pathlib import Path
31
+ from typing import Sequence
32
+
33
+ import numpy as np
34
+ import pandas as pd
35
+ from pydantic import BaseModel, Field, ValidationError
36
+ from torch.utils.data import Dataset
37
+ from tqdm import tqdm
38
+ from PIL import Image
39
+
40
+ from InternVLA.dataloader.gr00t_lerobot.video import get_all_frames, get_frames_by_timestamps
41
+
42
+ from InternVLA.dataloader.gr00t_lerobot.embodiment_tags import EmbodimentTag
43
+ from InternVLA.dataloader.gr00t_lerobot.schema import (
44
+ DatasetMetadata,
45
+ DatasetStatisticalValues,
46
+ LeRobotModalityMetadata,
47
+ LeRobotStateActionMetadata,
48
+ )
49
+ from InternVLA.dataloader.gr00t_lerobot.transform import ComposedModalityTransform
50
+
51
+ from functools import partial
52
+ from typing import Tuple, List
53
+ import pickle
54
+
55
+ LE_ROBOT_MODALITY_FILENAME = "meta/modality.json"
56
+ LE_ROBOT_EPISODE_FILENAME = "meta/episodes.jsonl"
57
+ LE_ROBOT_TASKS_FILENAME = "meta/tasks.jsonl"
58
+ LE_ROBOT_INFO_FILENAME = "meta/info.json"
59
+ LE_ROBOT_STATS_FILENAME = "meta/stats_gr00t.json"
60
+ LE_ROBOT_DATA_FILENAME = "data/*/*.parquet"
61
+ LE_ROBOT_STEPS_FILENAME = "meta/steps.pkl"
62
+ EPSILON = 5e-4
63
+
64
+ def calculate_dataset_statistics(parquet_paths: list[Path]) -> dict:
65
+ """Calculate the dataset statistics of all columns for a list of parquet files."""
66
+ # Dataset statistics
67
+ all_low_dim_data_list = []
68
+ # Collect all the data
69
+ # parquet_paths = parquet_paths[:3]
70
+ for parquet_path in tqdm(
71
+ sorted(list(parquet_paths)),
72
+ desc="Collecting all parquet files...",
73
+ ):
74
+ # Load the parquet file
75
+ parquet_data = pd.read_parquet(parquet_path)
76
+ parquet_data = parquet_data
77
+ all_low_dim_data_list.append(parquet_data)
78
+ all_low_dim_data = pd.concat(all_low_dim_data_list, axis=0)
79
+ # Compute dataset statistics
80
+ dataset_statistics = {}
81
+ for le_modality in all_low_dim_data.columns:
82
+ if le_modality not in ["states", "action"]:
83
+ continue
84
+ if le_modality.startswith("annotation."):
85
+ continue
86
+ print(f"Computing statistics for {le_modality}...")
87
+ np_data = np.vstack(
88
+ [np.asarray(x, dtype=np.float32) for x in all_low_dim_data[le_modality]]
89
+ )
90
+ dataset_statistics[le_modality] = {
91
+ "mean": np.mean(np_data, axis=0).tolist(),
92
+ "std": np.std(np_data, axis=0).tolist(),
93
+ "min": np.min(np_data, axis=0).tolist(),
94
+ "max": np.max(np_data, axis=0).tolist(),
95
+ "q01": np.quantile(np_data, 0.01, axis=0).tolist(),
96
+ "q99": np.quantile(np_data, 0.99, axis=0).tolist(),
97
+ }
98
+ return dataset_statistics
99
+
100
+
101
+ class ModalityConfig(BaseModel):
102
+ """Configuration for a modality."""
103
+
104
+ delta_indices: list[int]
105
+ """Delta indices to sample relative to the current index. The returned data will correspond to the original data at a sampled base index + delta indices."""
106
+ modality_keys: list[str]
107
+ """The keys to load for the modality in the dataset."""
108
+
109
+
110
+ class LeRobotSingleDataset(Dataset):
111
+ """
112
+ Base dataset class for LeRobot that supports sharding.
113
+ """
114
+ def __init__(
115
+ self,
116
+ dataset_path: Path | str,
117
+ modality_configs: dict[str, ModalityConfig],
118
+ embodiment_tag: str | EmbodimentTag,
119
+ video_backend: str = "decord",
120
+ video_backend_kwargs: dict | None = None,
121
+ transforms: ComposedModalityTransform | None = None,
122
+ delete_pause_frame: bool = False,
123
+ preload_all=True
124
+ ):
125
+ """
126
+ Initialize the dataset.
127
+
128
+ Args:
129
+ dataset_path (Path | str): The path to the dataset.
130
+ modality_configs (dict[str, ModalityConfig]): The configuration for each modality. The keys are the modality names, and the values are the modality configurations.
131
+ See `ModalityConfig` for more details.
132
+ video_backend (str): Backend for video reading.
133
+ video_backend_kwargs (dict): Keyword arguments for the video backend when initializing the video reader.
134
+ transforms (ComposedModalityTransform): The transforms to apply to the dataset.
135
+ embodiment_tag (EmbodimentTag): Overload the embodiment tag for the dataset. e.g. define it as "new_embodiment"
136
+ """
137
+ # first check if the path directory exists
138
+ if not Path(dataset_path).exists():
139
+ raise FileNotFoundError(f"Dataset path {dataset_path} does not exist")
140
+
141
+ self.delete_pause_frame = delete_pause_frame
142
+
143
+ self.modality_configs = modality_configs
144
+ self.video_backend = video_backend
145
+ self.video_backend_kwargs = video_backend_kwargs if video_backend_kwargs is not None else {}
146
+ self.transforms = (
147
+ transforms if transforms is not None else ComposedModalityTransform(transforms=[])
148
+ )
149
+
150
+ self._dataset_path = Path(dataset_path)
151
+ self._dataset_name = self._dataset_path.name
152
+ if isinstance(embodiment_tag, EmbodimentTag):
153
+ self.tag = embodiment_tag.value
154
+ else:
155
+ self.tag = embodiment_tag
156
+
157
+ self._metadata = self._get_metadata(EmbodimentTag(self.tag))
158
+
159
+ # LeRobot-specific config
160
+ self._lerobot_modality_meta = self._get_lerobot_modality_meta()
161
+ self._lerobot_info_meta = self._get_lerobot_info_meta()
162
+ self._data_path_pattern = self._get_data_path_pattern()
163
+ self._video_path_pattern = self._get_video_path_pattern()
164
+ self._chunk_size = self._get_chunk_size()
165
+ self._tasks = self._get_tasks()
166
+ self.curr_traj_data = None
167
+ self.curr_traj_id = None
168
+
169
+ self._trajectory_ids, self._trajectory_lengths = self._get_trajectories()
170
+ if preload_all:
171
+ self._preload_all_trajectories()
172
+ self.preload_all = True
173
+ else:
174
+ self.preload_all = False
175
+ self._modality_keys = self._get_modality_keys()
176
+ self._delta_indices = self._get_delta_indices()
177
+ self._all_steps = self._get_all_steps()
178
+ self.set_transforms_metadata(self.metadata)
179
+ self.set_epoch(0)
180
+
181
+ print(f"Initialized dataset {self.dataset_name} with {embodiment_tag}")
182
+
183
+
184
+ # Check if the dataset is valid
185
+ self._check_integrity()
186
+
187
+ @property
188
+ def dataset_path(self) -> Path:
189
+ """The path to the dataset that contains the METADATA_FILENAME file."""
190
+ return self._dataset_path
191
+
192
+ @property
193
+ def metadata(self) -> DatasetMetadata:
194
+ """The metadata for the dataset, loaded from metadata.json in the dataset directory"""
195
+ return self._metadata
196
+
197
+ @property
198
+ def trajectory_ids(self) -> np.ndarray:
199
+ """The trajectory IDs in the dataset, stored as a 1D numpy array of strings."""
200
+ return self._trajectory_ids
201
+
202
+ @property
203
+ def trajectory_lengths(self) -> np.ndarray:
204
+ """The trajectory lengths in the dataset, stored as a 1D numpy array of integers.
205
+ The order of the lengths is the same as the order of the trajectory IDs.
206
+ """
207
+ return self._trajectory_lengths
208
+
209
+ @property
210
+ def all_steps(self) -> list[tuple[int, int]]:
211
+ """The trajectory IDs and base indices for all steps in the dataset.
212
+ Example:
213
+ self.trajectory_ids: [0, 1, 2]
214
+ self.trajectory_lengths: [3, 2, 4]
215
+ return: [
216
+ ("traj_0", 0), ("traj_0", 1), ("traj_0", 2),
217
+ ("traj_1", 0), ("traj_1", 1),
218
+ ("traj_2", 0), ("traj_2", 1), ("traj_2", 2), ("traj_2", 3)
219
+ ]
220
+ """
221
+ return self._all_steps
222
+
223
+ @property
224
+ def modality_keys(self) -> dict:
225
+ """The modality keys for the dataset. The keys are the modality names, and the values are the keys for each modality.
226
+
227
+ Example: {
228
+ "video": ["video.image_side_0", "video.image_side_1"],
229
+ "state": ["state.eef_position", "state.eef_rotation"],
230
+ "action": ["action.eef_position", "action.eef_rotation"],
231
+ "language": ["language.human.task"],
232
+ "timestamp": ["timestamp"],
233
+ "reward": ["reward"],
234
+ }
235
+ """
236
+ return self._modality_keys
237
+
238
+ @property
239
+ def delta_indices(self) -> dict[str, np.ndarray]:
240
+ """The delta indices for the dataset. The keys are the modality.key, and the values are the delta indices for each modality.key."""
241
+ return self._delta_indices
242
+
243
+ @property
244
+ def dataset_name(self) -> str:
245
+ """The name of the dataset."""
246
+ return self._dataset_name
247
+
248
+ @property
249
+ def lerobot_modality_meta(self) -> LeRobotModalityMetadata:
250
+ """The metadata for the LeRobot dataset."""
251
+ return self._lerobot_modality_meta
252
+
253
+ @property
254
+ def lerobot_info_meta(self) -> dict:
255
+ """The metadata for the LeRobot dataset."""
256
+ return self._lerobot_info_meta
257
+
258
+ @property
259
+ def data_path_pattern(self) -> str:
260
+ """The path pattern for the LeRobot dataset."""
261
+ return self._data_path_pattern
262
+
263
+ @property
264
+ def video_path_pattern(self) -> str:
265
+ """The path pattern for the LeRobot dataset."""
266
+ return self._video_path_pattern
267
+
268
+ @property
269
+ def chunk_size(self) -> int:
270
+ """The chunk size for the LeRobot dataset."""
271
+ return self._chunk_size
272
+
273
+ @property
274
+ def tasks(self) -> pd.DataFrame:
275
+ """The tasks for the dataset."""
276
+ return self._tasks
277
+
278
+ def _get_metadata(self, embodiment_tag: EmbodimentTag) -> DatasetMetadata:
279
+ """Get the metadata for the dataset.
280
+
281
+ Returns:
282
+ dict: The metadata for the dataset.
283
+ """
284
+
285
+ # 1. Modality metadata
286
+ modality_meta_path = self.dataset_path / LE_ROBOT_MODALITY_FILENAME
287
+ assert (
288
+ modality_meta_path.exists()
289
+ ), f"Please provide a {LE_ROBOT_MODALITY_FILENAME} file in {self.dataset_path}"
290
+ # 1.1. State and action modalities
291
+ simplified_modality_meta: dict[str, dict] = {}
292
+ with open(modality_meta_path, "r") as f:
293
+ le_modality_meta = LeRobotModalityMetadata.model_validate(json.load(f))
294
+ for modality in ["state", "action"]:
295
+ simplified_modality_meta[modality] = {}
296
+ le_state_action_meta: dict[str, LeRobotStateActionMetadata] = getattr(
297
+ le_modality_meta, modality
298
+ )
299
+ for subkey in le_state_action_meta:
300
+ state_action_dtype = np.dtype(le_state_action_meta[subkey].dtype)
301
+ if np.issubdtype(state_action_dtype, np.floating):
302
+ continuous = True
303
+ else:
304
+ continuous = False
305
+ simplified_modality_meta[modality][subkey] = {
306
+ "absolute": le_state_action_meta[subkey].absolute,
307
+ "rotation_type": le_state_action_meta[subkey].rotation_type,
308
+ "shape": [
309
+ le_state_action_meta[subkey].end - le_state_action_meta[subkey].start
310
+ ],
311
+ "continuous": continuous,
312
+ }
313
+
314
+ # 1.2. Video modalities
315
+ le_info_path = self.dataset_path / LE_ROBOT_INFO_FILENAME
316
+ assert (
317
+ le_info_path.exists()
318
+ ), f"Please provide a {LE_ROBOT_INFO_FILENAME} file in {self.dataset_path}"
319
+ with open(le_info_path, "r") as f:
320
+ le_info = json.load(f)
321
+ simplified_modality_meta["video"] = {}
322
+ for new_key in le_modality_meta.video:
323
+ original_key = le_modality_meta.video[new_key].original_key
324
+ if original_key is None:
325
+ original_key = new_key
326
+ le_video_meta = le_info["features"][original_key]
327
+ height = le_video_meta["shape"][le_video_meta["names"].index("height")]
328
+ width = le_video_meta["shape"][le_video_meta["names"].index("width")]
329
+ # NOTE(FH): different lerobot dataset versions have different keys for the number of channels and fps
330
+ try:
331
+ channels = le_video_meta["shape"][le_video_meta["names"].index("channel")]
332
+ fps = le_video_meta["video_info"]["video.fps"]
333
+ except (ValueError, KeyError):
334
+ # channels = le_video_meta["shape"][le_video_meta["names"].index("channels")]
335
+ channels = le_video_meta["info"]["video.channels"]
336
+ fps = le_video_meta["info"]["video.fps"]
337
+ simplified_modality_meta["video"][new_key] = {
338
+ "resolution": [width, height],
339
+ "channels": channels,
340
+ "fps": fps,
341
+ }
342
+
343
+ # 2. Dataset statistics
344
+ stats_path = self.dataset_path / LE_ROBOT_STATS_FILENAME
345
+ try:
346
+ with open(stats_path, "r") as f:
347
+ le_statistics = json.load(f)
348
+ for stat in le_statistics.values():
349
+ DatasetStatisticalValues.model_validate(stat)
350
+ except (FileNotFoundError, ValidationError) as e:
351
+ print(f"Failed to load dataset statistics: {e}")
352
+ print(f"Calculating dataset statistics for {self.dataset_name}")
353
+ # Get all parquet files in the dataset paths
354
+ parquet_files = list((self.dataset_path).glob(LE_ROBOT_DATA_FILENAME))
355
+ le_statistics = calculate_dataset_statistics(parquet_files)
356
+ with open(stats_path, "w") as f:
357
+ json.dump(le_statistics, f, indent=4)
358
+ dataset_statistics = {}
359
+ for our_modality in ["state", "action"]:
360
+ dataset_statistics[our_modality] = {}
361
+ for subkey in simplified_modality_meta[our_modality]:
362
+ dataset_statistics[our_modality][subkey] = {}
363
+ state_action_meta = le_modality_meta.get_key_meta(f"{our_modality}.{subkey}")
364
+ assert isinstance(state_action_meta, LeRobotStateActionMetadata)
365
+ le_modality = state_action_meta.original_key
366
+ for stat_name in le_statistics[le_modality]:
367
+ indices = np.arange(
368
+ state_action_meta.start,
369
+ state_action_meta.end,
370
+ )
371
+ stat = np.array(le_statistics[le_modality][stat_name])
372
+ dataset_statistics[our_modality][subkey][stat_name] = stat[indices].tolist()
373
+
374
+ # 3. Full dataset metadata
375
+ metadata = DatasetMetadata(
376
+ statistics=dataset_statistics, # type: ignore
377
+ modalities=simplified_modality_meta, # type: ignore
378
+ embodiment_tag=embodiment_tag,
379
+ )
380
+
381
+ return metadata
382
+
383
+ def _get_trajectories(self) -> tuple[np.ndarray, np.ndarray]:
384
+ """Get the trajectories in the dataset."""
385
+ # Get trajectory lengths, IDs, and whitelist from dataset metadata
386
+ episode_path = self.dataset_path / LE_ROBOT_EPISODE_FILENAME
387
+ with open(episode_path, "r") as f:
388
+ episode_metadata = [json.loads(line) for line in f]
389
+ trajectory_ids = []
390
+ trajectory_lengths = []
391
+ for episode in episode_metadata:
392
+ trajectory_ids.append(episode["episode_index"])
393
+ trajectory_lengths.append(episode["length"])
394
+ return np.array(trajectory_ids), np.array(trajectory_lengths)
395
+
396
+ def _get_all_steps(self) -> list[tuple[int, int]]:
397
+ """Get the trajectory IDs and base indices for all steps in the dataset.
398
+
399
+ Returns:
400
+ list[tuple[str, int]]: A list of (trajectory_id, base_index) tuples.
401
+ """
402
+ # Create a hash key based on configuration to ensure cache validity
403
+ config_key = self._get_steps_config_key()
404
+
405
+ # Create a unique filename based on config_key
406
+ steps_filename = f"steps_{config_key}.pkl"
407
+ # @DUG
408
+ # fast get static steps @fangjing --> don't use hash to dynamic sample
409
+ steps_filename = "steps_7799a3080fbd.pkl"
410
+
411
+ steps_path = self.dataset_path / "meta" / steps_filename
412
+
413
+ # Try to load cached steps first
414
+ try:
415
+ if steps_path.exists():
416
+ with open(steps_path, "rb") as f:
417
+ cached_data = pickle.load(f)
418
+ return cached_data["steps"]
419
+ # # Verify the cached data matches current configuration
420
+ # if cached_data.get("config_key") == config_key:
421
+ # print(f"Loading cached steps from {steps_path}")
422
+ # return cached_data["steps"]
423
+ # else:
424
+ # print("Cached steps configuration mismatch, recomputing...")
425
+ except (FileNotFoundError, pickle.PickleError, KeyError) as e:
426
+ print(f"Failed to load cached steps: {e}")
427
+ print("Computing steps from scratch...")
428
+
429
+ # Compute steps using single process
430
+ all_steps = self._get_all_steps_single_process()
431
+
432
+ # Cache the computed steps with unique filename
433
+ try:
434
+ cache_data = {
435
+ "config_key": config_key,
436
+ "steps": all_steps,
437
+ "num_trajectories": len(self.trajectory_ids),
438
+ "total_steps": len(all_steps),
439
+ "computed_timestamp": pd.Timestamp.now().isoformat(),
440
+ "delete_pause_frame": self.delete_pause_frame,
441
+ }
442
+
443
+ # Ensure the meta directory exists
444
+ steps_path.parent.mkdir(parents=True, exist_ok=True)
445
+
446
+ with open(steps_path, "wb") as f:
447
+ pickle.dump(cache_data, f, protocol=pickle.HIGHEST_PROTOCOL)
448
+ print(f"Cached steps saved to {steps_path}")
449
+ except Exception as e:
450
+ print(f"Failed to cache steps: {e}")
451
+
452
+ return all_steps
453
+
454
+ def _get_steps_config_key(self) -> str:
455
+ """Generate a configuration key for steps caching."""
456
+ config_dict = {
457
+ "delete_pause_frame": self.delete_pause_frame,
458
+ "dataset_name": self.dataset_name,
459
+ }
460
+ # Create a hash of the configuration
461
+ config_str = str(sorted(config_dict.items()))
462
+ return hashlib.md5(config_str.encode()).hexdigest()[:12] #
463
+
464
+
465
+ def _get_all_steps_single_process(self) -> list[tuple[int, int]]:
466
+ """Original single-process implementation as fallback."""
467
+ all_steps: list[tuple[int, int]] = []
468
+ skipped_trajectories = 0
469
+ processed_trajectories = 0
470
+
471
+ # Check if language modality is configured
472
+ has_language_modality = 'language' in self.modality_keys and len(self.modality_keys['language']) > 0
473
+
474
+ for trajectory_id, trajectory_length in tqdm(zip(self.trajectory_ids, self.trajectory_lengths), total=len(self.trajectory_ids), desc="Getting All Step"):
475
+ data = self.get_trajectory_data(trajectory_id)
476
+ trajectory_skipped = False
477
+
478
+ # Check if trajectory has valid language instruction (if language modality is configured)
479
+ if has_language_modality:
480
+ self.curr_traj_data = data # Set current trajectory data for get_language to work
481
+ try:
482
+ language_instruction = self.get_language(trajectory_id, self.modality_keys['language'][0], 0)
483
+ print(language_instruction)
484
+ if not language_instruction or language_instruction[0] == "":
485
+ print(f"Skipping trajectory {trajectory_id} due to empty language instruction")
486
+ skipped_trajectories += 1
487
+ trajectory_skipped = True
488
+ continue
489
+ except Exception as e:
490
+ print(f"Skipping trajectory {trajectory_id} due to language retrieval error: {e}")
491
+ skipped_trajectories += 1
492
+ trajectory_skipped = True
493
+ continue
494
+
495
+ if not trajectory_skipped:
496
+ processed_trajectories += 1
497
+
498
+ self.delete_pause_frame = False # FIXME
499
+ if self.delete_pause_frame:
500
+ # Get position and gripper fields based on available columns
501
+ delta_position_values, gripper_values = self._get_position_and_gripper_values(data)
502
+ previous_gripper = gripper_values[0]
503
+ for base_index in range(trajectory_length):
504
+ if base_index >= len(delta_position_values) or base_index >= len(gripper_values):
505
+ break
506
+
507
+ # Check for translation change using the detected position fields
508
+ has_translation_change = np.any(np.abs(delta_position_values[base_index]) > EPSILON)
509
+ has_gripper_change = gripper_values[base_index] != (previous_gripper if base_index == 0 else gripper_values[base_index-1])
510
+
511
+ if has_translation_change or has_gripper_change:
512
+ all_steps.append((trajectory_id, base_index))
513
+ else:
514
+ for base_index in range(trajectory_length):
515
+ all_steps.append((trajectory_id, base_index))
516
+
517
+ # Print summary statistics
518
+ print(f"Single-process summary: Processed {processed_trajectories} trajectories, skipped {skipped_trajectories} empty trajectories")
519
+ print(f"Total steps: {len(all_steps)} from {len(self.trajectory_ids)} trajectories")
520
+
521
+ return all_steps
522
+
523
+ def _get_position_and_gripper_values(self, data: pd.DataFrame) -> tuple[list, list]:
524
+ """Get position and gripper values based on available columns in the dataset."""
525
+ # Get action keys from modality_keys
526
+ action_keys = self.modality_keys.get('action', [])
527
+
528
+ # Extract position data
529
+ delta_position_values = None
530
+ position_candidates = ['delta_eef_position']
531
+ coordinate_candidates = ['x', 'y', 'z']
532
+
533
+ # First try combined position fields
534
+ for pos_key in position_candidates:
535
+ full_key = f"action.{pos_key}"
536
+ if full_key in action_keys:
537
+ try:
538
+ # Get the lerobot key for this modality
539
+ le_action_cfg = self.lerobot_modality_meta.action
540
+ subkey = pos_key
541
+ if subkey in le_action_cfg:
542
+ le_key = le_action_cfg[subkey].original_key or subkey
543
+ if le_key in data.columns:
544
+ data_array = np.stack(data[le_key])
545
+ le_indices = np.arange(le_action_cfg[subkey].start, le_action_cfg[subkey].end)
546
+ filtered_data = data_array[:, le_indices]
547
+ delta_position_values = filtered_data.tolist()
548
+ break
549
+ except Exception:
550
+ continue
551
+
552
+ # If combined fields not found, try individual x,y,z coordinates
553
+ if delta_position_values is None:
554
+ x_data, y_data, z_data = None, None, None
555
+ for coord in coordinate_candidates:
556
+ full_key = f"action.{coord}"
557
+ if full_key in action_keys:
558
+ try:
559
+ le_action_cfg = self.lerobot_modality_meta.action
560
+ if coord in le_action_cfg:
561
+ le_key = le_action_cfg[coord].original_key or coord
562
+ if le_key in data.columns:
563
+ data_array = np.stack(data[le_key])
564
+ le_indices = np.arange(le_action_cfg[coord].start, le_action_cfg[coord].end)
565
+ coord_data = data_array[:, le_indices].flatten()
566
+ if coord == 'x':
567
+ x_data = coord_data
568
+ elif coord == 'y':
569
+ y_data = coord_data
570
+ elif coord == 'z':
571
+ z_data = coord_data
572
+ except Exception:
573
+ continue
574
+
575
+ if x_data is not None and y_data is not None and z_data is not None:
576
+ delta_position_values = np.column_stack((x_data, y_data, z_data)).tolist()
577
+
578
+ if delta_position_values is None:
579
+ # Fallback to the old hardcoded approach if metadata approach fails
580
+ if 'action.delta_eef_position' in data.columns:
581
+ delta_position_values = data['action.delta_eef_position'].to_numpy().tolist()
582
+ elif all(col in data.columns for col in ['action.x', 'action.y', 'action.z']):
583
+ x_vals = data['action.x'].to_numpy()
584
+ y_vals = data['action.y'].to_numpy()
585
+ z_vals = data['action.z'].to_numpy()
586
+ delta_position_values = np.column_stack((x_vals, y_vals, z_vals)).tolist()
587
+ else:
588
+ raise ValueError(f"No suitable position columns found. Available columns: {data.columns.tolist()}")
589
+
590
+ # Extract gripper data
591
+ gripper_values = None
592
+ gripper_candidates = ['gripper_close', 'gripper']
593
+
594
+ for grip_key in gripper_candidates:
595
+ full_key = f"action.{grip_key}"
596
+ if full_key in action_keys:
597
+ try:
598
+ le_action_cfg = self.lerobot_modality_meta.action
599
+ if grip_key in le_action_cfg:
600
+ le_key = le_action_cfg[grip_key].original_key or grip_key
601
+ if le_key in data.columns:
602
+ data_array = np.stack(data[le_key])
603
+ le_indices = np.arange(le_action_cfg[grip_key].start, le_action_cfg[grip_key].end)
604
+ gripper_data = data_array[:, le_indices].flatten()
605
+ gripper_values = gripper_data.tolist()
606
+ break
607
+ except Exception:
608
+ continue
609
+
610
+ if gripper_values is None:
611
+ # Fallback to the old hardcoded approach if metadata approach fails
612
+ if 'action.gripper_close' in data.columns:
613
+ gripper_values = data['action.gripper_close'].to_numpy().tolist()
614
+ elif 'action.gripper' in data.columns:
615
+ gripper_values = data['action.gripper'].to_numpy().tolist()
616
+ else:
617
+ raise ValueError(f"No suitable gripper columns found. Available columns: {data.columns.tolist()}")
618
+
619
+ return delta_position_values, gripper_values
620
+
621
+ def _get_modality_keys(self) -> dict:
622
+ """Get the modality keys for the dataset.
623
+ The keys are the modality names, and the values are the keys for each modality.
624
+ See property `modality_keys` for the expected format.
625
+ """
626
+ modality_keys = defaultdict(list)
627
+ for modality, config in self.modality_configs.items():
628
+ modality_keys[modality] = config.modality_keys
629
+ return modality_keys
630
+
631
+ def _get_delta_indices(self) -> dict[str, np.ndarray]:
632
+ """Restructure the delta indices to use modality.key as keys instead of just the modalities."""
633
+ delta_indices: dict[str, np.ndarray] = {}
634
+ for config in self.modality_configs.values():
635
+ for key in config.modality_keys:
636
+ delta_indices[key] = np.array(config.delta_indices)
637
+ return delta_indices
638
+
639
+ def _get_lerobot_modality_meta(self) -> LeRobotModalityMetadata:
640
+ """Get the metadata for the LeRobot dataset."""
641
+ modality_meta_path = self.dataset_path / LE_ROBOT_MODALITY_FILENAME
642
+ assert (
643
+ modality_meta_path.exists()
644
+ ), f"Please provide a {LE_ROBOT_MODALITY_FILENAME} file in {self.dataset_path}"
645
+ with open(modality_meta_path, "r") as f:
646
+ modality_meta = LeRobotModalityMetadata.model_validate(json.load(f))
647
+ return modality_meta
648
+
649
+ def _get_lerobot_info_meta(self) -> dict:
650
+ """Get the metadata for the LeRobot dataset."""
651
+ info_meta_path = self.dataset_path / LE_ROBOT_INFO_FILENAME
652
+ with open(info_meta_path, "r") as f:
653
+ info_meta = json.load(f)
654
+ return info_meta
655
+
656
+ def _get_data_path_pattern(self) -> str:
657
+ """Get the data path pattern for the LeRobot dataset."""
658
+ return self.lerobot_info_meta["data_path"]
659
+
660
+ def _get_video_path_pattern(self) -> str:
661
+ """Get the video path pattern for the LeRobot dataset."""
662
+ return self.lerobot_info_meta["video_path"]
663
+
664
+ def _get_chunk_size(self) -> int:
665
+ """Get the chunk size for the LeRobot dataset."""
666
+ return self.lerobot_info_meta["chunks_size"]
667
+
668
+ def _get_tasks(self) -> pd.DataFrame:
669
+ """Get the tasks for the dataset."""
670
+ tasks_path = self.dataset_path / LE_ROBOT_TASKS_FILENAME
671
+ with open(tasks_path, "r") as f:
672
+ tasks = [json.loads(line) for line in f]
673
+ df = pd.DataFrame(tasks)
674
+ return df.set_index("task_index")
675
+
676
+ def _check_integrity(self):
677
+ """Use the config to check if the keys are valid and detect silent data corruption."""
678
+ ERROR_MSG_HEADER = f"Error occurred in initializing dataset {self.dataset_name}:\n"
679
+
680
+ for modality_config in self.modality_configs.values():
681
+ for key in modality_config.modality_keys:
682
+ if key == "lapa_action" or key == "dream_actions":
683
+ continue # no need for any metadata for lapa actions because it comes normalized
684
+ # Check if the key is valid
685
+ try:
686
+ self.lerobot_modality_meta.get_key_meta(key)
687
+ except Exception as e:
688
+ raise ValueError(
689
+ ERROR_MSG_HEADER + f"Unable to find key {key} in modality metadata:\n{e}"
690
+ )
691
+
692
+ def set_transforms_metadata(self, metadata: DatasetMetadata):
693
+ """Set the metadata for the transforms. This is useful for transforms that need to know the metadata, such as the normalization values."""
694
+ self.transforms.set_metadata(metadata)
695
+
696
+ def set_epoch(self, epoch: int):
697
+ """Set the epoch for the dataset.
698
+
699
+ Args:
700
+ epoch (int): The epoch to set.
701
+ """
702
+ self.epoch = epoch
703
+
704
+ def __len__(self) -> int:
705
+ """Get the total number of data points in the dataset.
706
+
707
+ Returns:
708
+ int: the total number of data points in the dataset.
709
+ """
710
+ return len(self.all_steps)
711
+
712
+ def __str__(self) -> str:
713
+ """Get the description of the dataset."""
714
+ return f"{self.dataset_name} ({len(self)} steps)"
715
+
716
+
717
+ def __getitem__(self, index: int) -> dict:
718
+ """Get the data for a single step in a trajectory.
719
+
720
+ Args:
721
+ index (int): The index of the step to get.
722
+
723
+ Returns:
724
+ dict: The data for the step.
725
+ """
726
+ trajectory_id, base_index = self.all_steps[index]
727
+ data = self.get_step_data(trajectory_id, base_index)
728
+
729
+ # Process all video keys dynamically
730
+ images = []
731
+ for video_key in self.modality_keys["video"]:
732
+ image = data[video_key][0]
733
+
734
+ # Apply image cropping if enabled and the video key is base_view
735
+ # Note: crop_obs_camera functionality has been removed
736
+
737
+ image = Image.fromarray(image).resize((224, 224))
738
+ images.append(image)
739
+
740
+ # Get language and action data
741
+ language = data[self.modality_keys["language"][0]][0]
742
+ action = []
743
+ for action_key in self.modality_keys["action"]:
744
+ action.append(data[action_key])
745
+ action = np.concatenate(action, axis=1)
746
+
747
+ return dict(action=action, image=images, language=language)
748
+
749
+ def get_step_data(self, trajectory_id: int, base_index: int) -> dict:
750
+ """Get the RAW data for a single step in a trajectory. No transforms are applied.
751
+
752
+ Args:
753
+ trajectory_id (int): The name of the trajectory.
754
+ base_index (int): The base step index in the trajectory.
755
+
756
+ Returns:
757
+ dict: The RAW data for the step.
758
+
759
+ Example return:
760
+ {
761
+ "video": {
762
+ "video.image_side_0": [B, T, H, W, C],
763
+ "video.image_side_1": [B, T, H, W, C],
764
+ },
765
+ "state": {
766
+ "state.eef_position": [B, T, state_dim],
767
+ "state.eef_rotation": [B, T, state_dim],
768
+ },
769
+ "action": {
770
+ "action.eef_position": [B, T, action_dim],
771
+ "action.eef_rotation": [B, T, action_dim],
772
+ },
773
+ }
774
+ """
775
+ data = {}
776
+ # Get the data for all modalities
777
+ self.curr_traj_data = self.get_trajectory_data(trajectory_id)
778
+
779
+ for modality in self.modality_keys:
780
+ # Get the data corresponding to each key in the modality
781
+ for key in self.modality_keys[modality]:
782
+ data[key] = self.get_data_by_modality(trajectory_id, modality, key, base_index)
783
+ return data
784
+
785
+ def _preload_all_trajectories(self):
786
+ """预加载所有trajectories到内存"""
787
+ print("Preloading all trajectories...")
788
+ self._trajectory_cache = {}
789
+ for trajectory_id in tqdm(self.trajectory_ids, desc="Loading trajectories"):
790
+ chunk_index = self.get_episode_chunk(trajectory_id)
791
+ parquet_path = self.dataset_path / self.data_path_pattern.format(
792
+ episode_chunk=chunk_index, episode_index=trajectory_id
793
+ )
794
+ self._trajectory_cache[trajectory_id] = pd.read_parquet(parquet_path)
795
+ print(f"Preloaded {len(self._trajectory_cache)} trajectories")
796
+
797
+ def get_trajectory_data(self, trajectory_id: int) -> pd.DataFrame:
798
+ """Get the data for a trajectory."""
799
+
800
+ if self.preload_all:
801
+ return self._trajectory_cache[trajectory_id]
802
+ else:
803
+ if self.curr_traj_id == trajectory_id and self.curr_traj_data is not None:
804
+ return self.curr_traj_data
805
+ else:
806
+ chunk_index = self.get_episode_chunk(trajectory_id)
807
+ parquet_path = self.dataset_path / self.data_path_pattern.format(
808
+ episode_chunk=chunk_index, episode_index=trajectory_id
809
+ )
810
+ assert parquet_path.exists(), f"Parquet file not found at {parquet_path}"
811
+ return pd.read_parquet(parquet_path)
812
+
813
+ def get_trajectory_index(self, trajectory_id: int) -> int:
814
+ """Get the index of the trajectory in the dataset by the trajectory ID.
815
+ This is useful when you need to get the trajectory length or sampling weight corresponding to the trajectory ID.
816
+
817
+ Args:
818
+ trajectory_id (str): The ID of the trajectory.
819
+
820
+ Returns:
821
+ int: The index of the trajectory in the dataset.
822
+ """
823
+ trajectory_indices = np.where(self.trajectory_ids == trajectory_id)[0]
824
+ if len(trajectory_indices) != 1:
825
+ raise ValueError(
826
+ f"Error finding trajectory index for {trajectory_id}, found {trajectory_indices}"
827
+ )
828
+ return trajectory_indices[0]
829
+
830
+ def get_episode_chunk(self, ep_index: int) -> int:
831
+ """Get the chunk index for an episode index."""
832
+ return ep_index // self.chunk_size
833
+
834
+ def retrieve_data_and_pad(
835
+ self,
836
+ array: np.ndarray,
837
+ step_indices: np.ndarray,
838
+ max_length: int,
839
+ padding_strategy: str = "first_last",
840
+ ) -> np.ndarray:
841
+ """Retrieve the data from the dataset and pad it if necessary.
842
+ Args:
843
+ array (np.ndarray): The array to retrieve the data from.
844
+ step_indices (np.ndarray): The step indices to retrieve the data for.
845
+ max_length (int): The maximum length of the data.
846
+ padding_strategy (str): The padding strategy, either "first" or "last".
847
+ """
848
+ # Get the padding indices
849
+ front_padding_indices = step_indices < 0
850
+ end_padding_indices = step_indices >= max_length
851
+ padding_positions = np.logical_or(front_padding_indices, end_padding_indices)
852
+ # Retrieve the data with the non-padding indices
853
+ # If there exists some padding, Given T step_indices, the shape of the retrieved data will be (T', ...) where T' < T
854
+ raw_data = array[step_indices[~padding_positions]]
855
+ assert isinstance(raw_data, np.ndarray), f"{type(raw_data)=}"
856
+ # This is the shape of the output, (T, ...)
857
+ if raw_data.ndim == 1:
858
+ expected_shape = (len(step_indices),)
859
+ else:
860
+ expected_shape = (len(step_indices), *array.shape[1:])
861
+
862
+ # Pad the data
863
+ output = np.zeros(expected_shape)
864
+ # Assign the non-padded data
865
+ output[~padding_positions] = raw_data
866
+ # If there exists some padding, pad the data
867
+ if padding_positions.any():
868
+ if padding_strategy == "first_last":
869
+ # Use first / last step data to pad
870
+ front_padding_data = array[0]
871
+ end_padding_data = array[-1]
872
+ output[front_padding_indices] = front_padding_data
873
+ output[end_padding_indices] = end_padding_data
874
+ elif padding_strategy == "zero":
875
+ # Use zero padding
876
+ output[padding_positions] = 0
877
+ else:
878
+ raise ValueError(f"Invalid padding strategy: {padding_strategy}")
879
+ return output
880
+
881
+ def get_video_path(self, trajectory_id: int, key: str) -> Path:
882
+ chunk_index = self.get_episode_chunk(trajectory_id)
883
+ original_key = self.lerobot_modality_meta.video[key].original_key
884
+ if original_key is None:
885
+ original_key = key
886
+ video_filename = self.video_path_pattern.format(
887
+ episode_chunk=chunk_index, episode_index=trajectory_id, video_key=original_key
888
+ )
889
+ return self.dataset_path / video_filename
890
+
891
+ def get_video(
892
+ self,
893
+ trajectory_id: int,
894
+ key: str,
895
+ base_index: int,
896
+ ) -> np.ndarray:
897
+ """Get the video frames for a trajectory by a base index.
898
+
899
+ Args:
900
+ dataset (BaseSingleDataset): The dataset to retrieve the data from.
901
+ trajectory_id (str): The ID of the trajectory.
902
+ key (str): The key of the video.
903
+ base_index (int): The base index of the trajectory.
904
+
905
+ Returns:
906
+ np.ndarray: The video frames for the trajectory and frame indices. Shape: (T, H, W, C)
907
+ """
908
+ # Get the step indices
909
+ step_indices = self.delta_indices[key] + base_index
910
+ # print(f"{step_indices=}")
911
+ # Get the trajectory index
912
+ trajectory_index = self.get_trajectory_index(trajectory_id)
913
+ # Ensure the indices are within the valid range
914
+ # This is equivalent to padding the video with extra frames at the beginning and end
915
+ step_indices = np.maximum(step_indices, 0)
916
+ step_indices = np.minimum(step_indices, self.trajectory_lengths[trajectory_index] - 1)
917
+ assert key.startswith("video."), f"Video key must start with 'video.', got {key}"
918
+ # Get the sub-key
919
+ key = key.replace("video.", "")
920
+ video_path = self.get_video_path(trajectory_id, key)
921
+ # Get the action/state timestamps for each frame in the video
922
+ assert self.curr_traj_data is not None, f"No data found for {trajectory_id}"
923
+ assert "timestamp" in self.curr_traj_data.columns, f"No timestamp found in {trajectory_id}"
924
+ timestamp: np.ndarray = self.curr_traj_data["timestamp"].to_numpy()
925
+ # Get the corresponding video timestamps from the step indices
926
+ video_timestamp = timestamp[step_indices]
927
+
928
+ return get_frames_by_timestamps(
929
+ video_path.as_posix(),
930
+ video_timestamp,
931
+ video_backend=self.video_backend,
932
+ video_backend_kwargs=self.video_backend_kwargs,
933
+ step_indices_for_assert=step_indices,
934
+ )
935
+
936
+ def get_state_or_action(
937
+ self,
938
+ trajectory_id: int,
939
+ modality: str,
940
+ key: str,
941
+ base_index: int,
942
+ ) -> np.ndarray:
943
+ """Get the state or action data for a trajectory by a base index.
944
+ If the step indices are out of range, pad with the data:
945
+ if the data is stored in absolute format, pad with the first or last step data;
946
+ otherwise, pad with zero.
947
+
948
+ Args:
949
+ dataset (BaseSingleDataset): The dataset to retrieve the data from.
950
+ trajectory_id (int): The ID of the trajectory.
951
+ modality (str): The modality of the data.
952
+ key (str): The key of the data.
953
+ base_index (int): The base index of the trajectory.
954
+
955
+ Returns:
956
+ np.ndarray: The data for the trajectory and step indices.
957
+ """
958
+ # Get the step indices
959
+ step_indices = self.delta_indices[key] + base_index
960
+ # Get the trajectory index
961
+ trajectory_index = self.get_trajectory_index(trajectory_id)
962
+ # Get the maximum length of the trajectory
963
+ max_length = self.trajectory_lengths[trajectory_index]
964
+ assert key.startswith(modality + "."), f"{key} must start with {modality + '.'}, got {key}"
965
+ # Get the sub-key, e.g. state.joint_angles -> joint_angles
966
+ key = key.replace(modality + ".", "")
967
+ # Get the lerobot key
968
+ le_state_or_action_cfg = getattr(self.lerobot_modality_meta, modality)
969
+ le_key = le_state_or_action_cfg[key].original_key
970
+ if le_key is None:
971
+ le_key = key
972
+ # Get the data array, shape: (T, D)
973
+ assert self.curr_traj_data is not None, f"No data found for {trajectory_id}"
974
+ assert le_key in self.curr_traj_data.columns, f"No {le_key} found in {trajectory_id}"
975
+ data_array: np.ndarray = np.stack(self.curr_traj_data[le_key]) # type: ignore
976
+ assert data_array.ndim == 2, f"Expected 2D array, got key {le_key} is{data_array.shape} array"
977
+ le_indices = np.arange(
978
+ le_state_or_action_cfg[key].start,
979
+ le_state_or_action_cfg[key].end,
980
+ )
981
+ data_array = data_array[:, le_indices]
982
+ # Get the state or action configuration
983
+ state_or_action_cfg = getattr(self.metadata.modalities, modality)[key]
984
+
985
+ # Pad the data
986
+ return self.retrieve_data_and_pad(
987
+ array=data_array,
988
+ step_indices=step_indices,
989
+ max_length=max_length,
990
+ padding_strategy="first_last" if state_or_action_cfg.absolute else "zero",
991
+ # padding_strategy="first_last", # HACK for realdata
992
+ )
993
+
994
+ def get_language(
995
+ self,
996
+ trajectory_id: int,
997
+ key: str,
998
+ base_index: int,
999
+ ) -> list[str]:
1000
+ """Get the language annotation data for a trajectory by step indices.
1001
+
1002
+ Args:
1003
+ dataset (BaseSingleDataset): The dataset to retrieve the data from.
1004
+ trajectory_id (int): The ID of the trajectory.
1005
+ key (str): The key of the annotation.
1006
+ base_index (int): The base index of the trajectory.
1007
+
1008
+ Returns:
1009
+ list[str]: The annotation data for the trajectory and step indices. If no matching data is found, return empty strings.
1010
+ """
1011
+ assert self.curr_traj_data is not None, f"No data found for {trajectory_id}"
1012
+ # Get the step indices
1013
+ step_indices = self.delta_indices[key] + base_index
1014
+ # Get the trajectory index
1015
+ trajectory_index = self.get_trajectory_index(trajectory_id)
1016
+ # Get the maximum length of the trajectory
1017
+ max_length = self.trajectory_lengths[trajectory_index]
1018
+ # Get the end times corresponding to the closest indices
1019
+ step_indices = np.maximum(step_indices, 0)
1020
+ step_indices = np.minimum(step_indices, max_length - 1)
1021
+ # Get the annotations
1022
+ task_indices: list[int] = []
1023
+ assert key.startswith(
1024
+ "annotation."
1025
+ ), f"Language key must start with 'annotation.', got {key}"
1026
+ subkey = key.replace("annotation.", "")
1027
+ annotation_meta = self.lerobot_modality_meta.annotation
1028
+ assert annotation_meta is not None, f"Annotation metadata is None for {subkey}"
1029
+ assert (
1030
+ subkey in annotation_meta
1031
+ ), f"Annotation key {subkey} not found in metadata, available annotation keys: {annotation_meta.keys()}"
1032
+ subkey_meta = annotation_meta[subkey]
1033
+ original_key = subkey_meta.original_key
1034
+ if original_key is None:
1035
+ original_key = key
1036
+ for i in range(len(step_indices)):
1037
+ task_indices.append(self.curr_traj_data[original_key][step_indices[i]].item())
1038
+ return self.tasks.loc[task_indices]["description"].tolist() # 适配zhenyu脚本
1039
+
1040
+ def get_data_by_modality(
1041
+ self,
1042
+ trajectory_id: int,
1043
+ modality: str,
1044
+ key: str,
1045
+ base_index: int,
1046
+ ):
1047
+ """Get the data corresponding to the modality for a trajectory by a base index.
1048
+ This method will call the corresponding helper method based on the modality.
1049
+ See the helper methods for more details.
1050
+ NOTE: For the language modality, the data is padded with empty strings if no matching data is found.
1051
+
1052
+ Args:
1053
+ dataset (BaseSingleDataset): The dataset to retrieve the data from.
1054
+ trajectory_id (int): The ID of the trajectory.
1055
+ modality (str): The modality of the data.
1056
+ key (str): The key of the data.
1057
+ base_index (int): The base index of the trajectory.
1058
+ """
1059
+ if modality == "video":
1060
+ return self.get_video(trajectory_id, key, base_index)
1061
+ elif modality == "state" or modality == "action":
1062
+ return self.get_state_or_action(trajectory_id, modality, key, base_index)
1063
+ elif modality == "language":
1064
+ return self.get_language(trajectory_id, key, base_index)
1065
+ else:
1066
+ raise ValueError(f"Invalid modality: {modality}")
1067
+
1068
+ def save_dataset_statistics(self, save_path: Path | str, format: str = "json") -> None:
1069
+ """
1070
+ Save dataset statistics to specified path in the required format.
1071
+ Only includes statistics for keys that are actually used in the dataset.
1072
+ Gripper-related keys will be placed at the end.
1073
+
1074
+ Args:
1075
+ save_path (Path | str): Path to save the statistics file
1076
+ format (str): Save format, currently only supports "json"
1077
+ """
1078
+ save_path = Path(save_path)
1079
+ save_path.parent.mkdir(parents=True, exist_ok=True)
1080
+
1081
+ # Build the data structure to save
1082
+ statistics_data = {}
1083
+
1084
+ # Get used modality keys
1085
+ used_action_keys, used_state_keys = get_used_modality_keys(self.modality_keys)
1086
+
1087
+ # Organize statistics by tag
1088
+ tag = self.tag
1089
+ tag_stats = {}
1090
+
1091
+ # Process action statistics (only for used keys)
1092
+ if hasattr(self.metadata.statistics, 'action') and self.metadata.statistics.action:
1093
+ action_stats = self.metadata.statistics.action
1094
+
1095
+ # Filter to only include used action keys and reorder: non-gripper first, gripper last
1096
+ non_gripper_keys = []
1097
+ gripper_keys = []
1098
+
1099
+ for key in action_stats.keys():
1100
+ if key in used_action_keys:
1101
+ if "gripper" in key.lower():
1102
+ gripper_keys.append(key)
1103
+ else:
1104
+ non_gripper_keys.append(key)
1105
+
1106
+ # Reorder: non-gripper first, gripper last
1107
+ reordered_keys = non_gripper_keys + gripper_keys
1108
+
1109
+ filtered_action_stats = {}
1110
+ for key in reordered_keys:
1111
+ filtered_action_stats[key] = action_stats[key]
1112
+
1113
+ if filtered_action_stats:
1114
+ # Combine statistics from filtered action sub-keys
1115
+ combined_action_stats = combine_modality_stats(filtered_action_stats)
1116
+
1117
+ # Add mask field: based on whether it's gripper or not
1118
+ mask = generate_action_mask_for_used_keys(
1119
+ self.metadata.modalities.action, filtered_action_stats.keys()
1120
+ )
1121
+ combined_action_stats["mask"] = mask
1122
+
1123
+ tag_stats["action"] = combined_action_stats
1124
+
1125
+ # Process state statistics (only for used keys)
1126
+ if hasattr(self.metadata.statistics, 'state') and self.metadata.statistics.state:
1127
+ state_stats = self.metadata.statistics.state
1128
+
1129
+ # Filter to only include used state keys, optionally reorder gripper to end
1130
+ non_gripper_keys = []
1131
+ gripper_keys = []
1132
+
1133
+ for key in state_stats.keys():
1134
+ if key in used_state_keys:
1135
+ if "gripper" in key.lower():
1136
+ gripper_keys.append(key)
1137
+ else:
1138
+ non_gripper_keys.append(key)
1139
+
1140
+ # Reorder: non-gripper first, gripper last
1141
+ reordered_keys = non_gripper_keys + gripper_keys
1142
+
1143
+ filtered_state_stats = {}
1144
+ for key in reordered_keys:
1145
+ filtered_state_stats[key] = state_stats[key]
1146
+
1147
+ if filtered_state_stats:
1148
+ combined_state_stats = combine_modality_stats(filtered_state_stats)
1149
+ tag_stats["state"] = combined_state_stats
1150
+
1151
+ # Add dataset counts
1152
+ tag_stats["num_transitions"] = len(self)
1153
+ tag_stats["num_trajectories"] = len(self.trajectory_ids)
1154
+
1155
+ statistics_data[tag] = tag_stats
1156
+
1157
+ # Save as JSON file
1158
+ if format.lower() == "json":
1159
+ if not str(save_path).endswith('.json'):
1160
+ save_path = save_path.with_suffix('.json')
1161
+ with open(save_path, 'w', encoding='utf-8') as f:
1162
+ json.dump(statistics_data, f, indent=2, ensure_ascii=False)
1163
+ else:
1164
+ raise ValueError(f"Unsupported format: {format}. Currently only 'json' is supported.")
1165
+
1166
+ print(f"Single dataset statistics saved to: {save_path}")
1167
+ print(f"Used action keys (reordered): {list(used_action_keys)}")
1168
+ print(f"Used state keys (reordered): {list(used_state_keys)}")
1169
+
1170
+
1171
+ class CachedLeRobotSingleDataset(LeRobotSingleDataset):
1172
+ def __init__(self, img_resize: tuple[int, int] | None = None, *args, **kwargs):
1173
+ """
1174
+ This class caches the video frames for each trajectory and key.
1175
+ It is recommended to use this class if the video frames need to be accessed multiple times.
1176
+
1177
+ Args:
1178
+ resize_img (tuple[int, int], optional): The size to resize the video frames to reduce memory usage.
1179
+ """
1180
+ # Convert img_resize to tuple if it is not already
1181
+ if img_resize is not None and not isinstance(img_resize, tuple):
1182
+ img_resize = tuple(img_resize)
1183
+ assert len(img_resize) == 2, f"Expected tuple of length 2, got {img_resize}"
1184
+ self.img_resize = img_resize
1185
+
1186
+ # Initialize img_resize attribute first to ensure it exists
1187
+ super().__init__(*args, **kwargs)
1188
+ cached_frames: dict[str, np.ndarray] = {}
1189
+
1190
+ for key in self.modality_keys["video"]:
1191
+ all_frames = []
1192
+ original_key = key
1193
+ key = key.replace("video.", "")
1194
+ for trajectory_id, trajectory_length in tqdm(
1195
+ zip(self.trajectory_ids, self.trajectory_lengths),
1196
+ total=len(self.trajectory_ids),
1197
+ desc=f"Caching {key} frames",
1198
+ ):
1199
+ video_path = self.get_video_path(trajectory_id, key)
1200
+ frames = get_all_frames(
1201
+ video_path.as_posix(),
1202
+ video_backend=self.video_backend,
1203
+ video_backend_kwargs=self.video_backend_kwargs,
1204
+ resize_size=img_resize,
1205
+ )
1206
+ assert frames.ndim == 4, f"Expected 4D array, got {frames.shape} array"
1207
+ assert frames.shape[3] == 3, f"Expected 3 channels, got {frames.shape[3]} channels"
1208
+
1209
+ # Apply image cropping if enabled and the video key is base_view
1210
+ # Note: crop_obs_camera functionality has been removed
1211
+
1212
+ # assert (
1213
+ # frames.shape[0] == trajectory_length
1214
+ # ), f"Expected {trajectory_length} frames, got {frames.shape[0]} frames"
1215
+ all_frames.append(frames)
1216
+ cached_frames[key] = np.concatenate(all_frames, axis=0)
1217
+ print(f"{key}: {cached_frames[key].shape}")
1218
+ self.cached_frames = cached_frames
1219
+ self.start_indices = np.cumsum(self.trajectory_lengths) - self.trajectory_lengths
1220
+
1221
+ def get_video(self, trajectory_id: int, key: str, base_index: int) -> np.ndarray:
1222
+ step_indices = self.delta_indices[key] + base_index
1223
+ # Get the trajectory index
1224
+ trajectory_index = self.get_trajectory_index(trajectory_id)
1225
+ # Ensure the indices are within the valid range
1226
+ # This is equivalent to padding the video with extra frames at the beginning and end
1227
+ step_indices = np.maximum(step_indices, 0)
1228
+ step_indices = np.minimum(step_indices, self.trajectory_lengths[trajectory_index] - 1)
1229
+ assert key.startswith("video."), f"Video key must start with 'video.', got {key}"
1230
+ # Get the sub-key
1231
+ key = key.replace("video.", "")
1232
+ # Calculate the absolute indices
1233
+ absolute_indices = self.start_indices[trajectory_index] + step_indices
1234
+ return self.cached_frames[key][absolute_indices]
1235
+
1236
+ def get_step_data(self, trajectory_id: int, base_index: int) -> dict:
1237
+ """Get the RAW data for a single step. No transforms are applied.
1238
+
1239
+ Args:
1240
+ trajectory_id (str): The ID of the trajectory.
1241
+ base_index (int): The base index of the step.
1242
+
1243
+ Returns:
1244
+ dict: The data for the step.
1245
+ """
1246
+ data = {}
1247
+ self.curr_traj_data = self.get_trajectory_data(trajectory_id)
1248
+ # Get the data for all modalities
1249
+ for modality in self.modality_keys:
1250
+ # Get the data corresponding to each key in the modality
1251
+ for key in self.modality_keys[modality]:
1252
+ data[key] = self.get_data_by_modality(trajectory_id, modality, key, base_index)
1253
+ return data
1254
+
1255
+ def set_transforms_metadata(self, metadata: DatasetMetadata):
1256
+ """Set the metadata for the transforms. This is useful for transforms that need to know the metadata, such as the normalization values."""
1257
+ if self.img_resize is not None:
1258
+ all_video_keys = [key for key in self.modality_keys["video"]]
1259
+ for key in metadata.modalities.video:
1260
+ if key in all_video_keys:
1261
+ metadata.modalities.video[key].resolution = self.img_resize
1262
+ super().set_transforms_metadata(metadata)
1263
+
1264
+
1265
+ def safe_hash(input_tuple):
1266
+ # keep 128 bits of the hash
1267
+ tuple_string = repr(input_tuple).encode("utf-8")
1268
+ sha256 = hashlib.sha256()
1269
+ sha256.update(tuple_string)
1270
+
1271
+ seed = int(sha256.hexdigest(), 16)
1272
+
1273
+ return seed & 0xFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
1274
+
1275
+
1276
+ class MixtureSpecElement(BaseModel):
1277
+ dataset_path: list[Path] | Path = Field(..., description="The path to the dataset.")
1278
+ dataset_weight: float = Field(..., description="The weight of the dataset in the mixture.")
1279
+ distribute_weights: bool = Field(
1280
+ default=False,
1281
+ description="Whether to distribute the weights of the dataset across all the paths. If True, the weights will be evenly distributed across all the paths.",
1282
+ )
1283
+
1284
+
1285
+ # Helper functions for dataset statistics
1286
+
1287
+ def combine_modality_stats(modality_stats: dict) -> dict:
1288
+ """
1289
+ Combine statistics from all sub-keys under a modality.
1290
+
1291
+ Args:
1292
+ modality_stats (dict): Statistics for a modality, containing multiple sub-keys.
1293
+ Each sub-key contains DatasetStatisticalValues object.
1294
+
1295
+ Returns:
1296
+ dict: Combined statistics
1297
+ """
1298
+ combined_stats = {
1299
+ "mean": [],
1300
+ "std": [],
1301
+ "max": [],
1302
+ "min": [],
1303
+ "q01": [],
1304
+ "q99": []
1305
+ }
1306
+
1307
+ # Combine statistics in sub-key order
1308
+ for subkey in modality_stats.keys():
1309
+ subkey_stats = modality_stats[subkey] # This is a DatasetStatisticalValues object
1310
+
1311
+ # Convert DatasetStatisticalValues to dict-like access
1312
+ for stat_name in ["mean", "std", "max", "min", "q01", "q99"]:
1313
+ stat_value = getattr(subkey_stats, stat_name)
1314
+ if isinstance(stat_value, (list, tuple)):
1315
+ combined_stats[stat_name].extend(stat_value)
1316
+ else:
1317
+ # Handle NDArray case - convert to list
1318
+ if hasattr(stat_value, 'tolist'):
1319
+ combined_stats[stat_name].extend(stat_value.tolist())
1320
+ else:
1321
+ combined_stats[stat_name].append(float(stat_value))
1322
+
1323
+ return combined_stats
1324
+
1325
+ def generate_action_mask_for_used_keys(action_modalities: dict, used_action_keys_ordered) -> list[bool]:
1326
+ """
1327
+ Generate mask based on action modalities, but only for used keys.
1328
+ Gripper-related are False, others are True.
1329
+
1330
+ Args:
1331
+ action_modalities (dict): Configuration information for action modalities.
1332
+ used_action_keys_ordered: Iterable of actually used action keys in the correct order.
1333
+
1334
+ Returns:
1335
+ list[bool]: List of mask values
1336
+ """
1337
+ mask = []
1338
+
1339
+ # Generate mask in the same order as the statistics were combined
1340
+ for subkey in used_action_keys_ordered:
1341
+ if subkey in action_modalities:
1342
+ subkey_config = action_modalities[subkey]
1343
+
1344
+ # Get dimension count from shape
1345
+ if hasattr(subkey_config, 'shape') and len(subkey_config.shape) > 0:
1346
+ dim_count = subkey_config.shape[0]
1347
+ else:
1348
+ dim_count = 1
1349
+
1350
+ # Check if it's gripper-related
1351
+ is_gripper = "gripper" in subkey.lower()
1352
+
1353
+ # Generate mask value for each dimension
1354
+ for _ in range(dim_count):
1355
+ mask.append(not is_gripper) # gripper is False, others are True
1356
+
1357
+ return mask
1358
+
1359
+ def get_used_modality_keys(modality_keys: dict) -> tuple[set, set]:
1360
+ """Extract used action and state keys from modality configuration."""
1361
+ used_action_keys = set()
1362
+ used_state_keys = set()
1363
+
1364
+ # Extract action keys (remove "action." prefix)
1365
+ for action_key in modality_keys.get("action", []):
1366
+ if action_key.startswith("action."):
1367
+ clean_key = action_key.replace("action.", "")
1368
+ used_action_keys.add(clean_key)
1369
+
1370
+ # Extract state keys (remove "state." prefix)
1371
+ for state_key in modality_keys.get("state", []):
1372
+ if state_key.startswith("state."):
1373
+ clean_key = state_key.replace("state.", "")
1374
+ used_state_keys.add(clean_key)
1375
+
1376
+ return used_action_keys, used_state_keys
1377
+
1378
+ class LeRobotMixtureDataset(Dataset):
1379
+ """
1380
+ A mixture of multiple datasets. This class samples a single dataset based on the dataset weights and then calls the `__getitem__` method of the sampled dataset.
1381
+ It is recommended to modify the single dataset class instead of this class.
1382
+ """
1383
+
1384
+ def __init__(
1385
+ self,
1386
+ data_mixture: Sequence[tuple[LeRobotSingleDataset, float]],
1387
+ mode: str,
1388
+ balance_dataset_weights: bool = True,
1389
+ balance_trajectory_weights: bool = True,
1390
+ seed: int = 42,
1391
+ metadata_config: dict = {
1392
+ "percentile_mixing_method": "min_max",
1393
+ },
1394
+ ):
1395
+ """
1396
+ Initialize the mixture dataset.
1397
+
1398
+ Args:
1399
+ data_mixture (list[tuple[LeRobotSingleDataset, float]]): Datasets and their corresponding weights.
1400
+ mode (str): If "train", __getitem__ will return different samples every epoch; if "val" or "test", __getitem__ will return the same sample every epoch.
1401
+ balance_dataset_weights (bool): If True, the weight of dataset will be multiplied by the total trajectory length of each dataset.
1402
+ balance_trajectory_weights (bool): If True, sample trajectories within a dataset weighted by their length; otherwise, use equal weighting.
1403
+ seed (int): Random seed for sampling.
1404
+ """
1405
+ datasets: list[LeRobotSingleDataset] = []
1406
+ dataset_sampling_weights: list[float] = []
1407
+ for dataset, weight in data_mixture:
1408
+ # Check if dataset is valid and has data
1409
+ if len(dataset) == 0:
1410
+ print(f"Warning: Skipping empty dataset {dataset.dataset_name}")
1411
+ continue
1412
+ datasets.append(dataset)
1413
+ dataset_sampling_weights.append(weight)
1414
+
1415
+ if len(datasets) == 0:
1416
+ raise ValueError("No valid datasets found in the mixture. All datasets are empty.")
1417
+
1418
+ self.datasets = datasets
1419
+ self.balance_dataset_weights = balance_dataset_weights
1420
+ self.balance_trajectory_weights = balance_trajectory_weights
1421
+ self.seed = seed
1422
+ self.mode = mode
1423
+
1424
+ # Set properties for sampling
1425
+
1426
+ # 1. Dataset lengths
1427
+ self._dataset_lengths = np.array([len(dataset) for dataset in self.datasets])
1428
+ print(f"Dataset lengths: {self._dataset_lengths}")
1429
+
1430
+ # 2. Dataset sampling weights
1431
+ self._dataset_sampling_weights = np.array(dataset_sampling_weights)
1432
+
1433
+ if self.balance_dataset_weights:
1434
+ self._dataset_sampling_weights *= self._dataset_lengths
1435
+
1436
+ # Check for zero or negative weights before normalization
1437
+ if np.any(self._dataset_sampling_weights <= 0):
1438
+ print(f"Warning: Found zero or negative sampling weights: {self._dataset_sampling_weights}")
1439
+ # Set minimum weight to prevent division issues
1440
+ self._dataset_sampling_weights = np.maximum(self._dataset_sampling_weights, 1e-8)
1441
+
1442
+ # Normalize weights
1443
+ weights_sum = self._dataset_sampling_weights.sum()
1444
+ if weights_sum == 0 or np.isnan(weights_sum):
1445
+ print(f"Error: Invalid weights sum: {weights_sum}")
1446
+ # Fallback to equal weights
1447
+ self._dataset_sampling_weights = np.ones(len(self.datasets)) / len(self.datasets)
1448
+ print(f"Fallback to equal weights")
1449
+ else:
1450
+ self._dataset_sampling_weights /= weights_sum
1451
+
1452
+ # 3. Trajectory sampling weights
1453
+ self._trajectory_sampling_weights: list[np.ndarray] = []
1454
+ for i, dataset in enumerate(self.datasets):
1455
+ trajectory_sampling_weights = np.ones(len(dataset.trajectory_lengths))
1456
+ if self.balance_trajectory_weights:
1457
+ trajectory_sampling_weights *= dataset.trajectory_lengths
1458
+
1459
+ # Check for zero or negative weights before normalization
1460
+ if np.any(trajectory_sampling_weights <= 0):
1461
+ print(f"Warning: Dataset {i} has zero or negative trajectory weights")
1462
+ trajectory_sampling_weights = np.maximum(trajectory_sampling_weights, 1e-8)
1463
+
1464
+ # Normalize weights
1465
+ weights_sum = trajectory_sampling_weights.sum()
1466
+ if weights_sum == 0 or np.isnan(weights_sum):
1467
+ print(f"Error: Dataset {i} has invalid trajectory weights sum: {weights_sum}")
1468
+ # Fallback to equal weights
1469
+ trajectory_sampling_weights = np.ones(len(dataset.trajectory_lengths)) / len(dataset.trajectory_lengths)
1470
+ else:
1471
+ trajectory_sampling_weights /= weights_sum
1472
+
1473
+ self._trajectory_sampling_weights.append(trajectory_sampling_weights)
1474
+
1475
+ # 4. Primary dataset indices
1476
+ self._primary_dataset_indices = np.array(dataset_sampling_weights) == 1.0
1477
+ if not np.any(self._primary_dataset_indices):
1478
+ print(f"Warning: No dataset with weight 1.0 found. Original weights: {dataset_sampling_weights}")
1479
+ # Fallback: use the dataset(s) with maximum weight as primary
1480
+ max_weight = max(dataset_sampling_weights)
1481
+ self._primary_dataset_indices = np.array(dataset_sampling_weights) == max_weight
1482
+ print(f"Using datasets with maximum weight {max_weight} as primary: {self._primary_dataset_indices}")
1483
+
1484
+ if not np.any(self._primary_dataset_indices):
1485
+ # This should never happen, but just in case
1486
+ print("Error: Still no primary dataset found. Using first dataset as primary.")
1487
+ self._primary_dataset_indices = np.zeros(len(self.datasets), dtype=bool)
1488
+ self._primary_dataset_indices[0] = True
1489
+
1490
+ # Set the epoch and sample the first epoch
1491
+ self.set_epoch(0)
1492
+
1493
+ self.update_metadata(metadata_config)
1494
+
1495
+ @property
1496
+ def dataset_lengths(self) -> np.ndarray:
1497
+ """The lengths of each dataset."""
1498
+ return self._dataset_lengths
1499
+
1500
+ @property
1501
+ def dataset_sampling_weights(self) -> np.ndarray:
1502
+ """The sampling weights for each dataset."""
1503
+ return self._dataset_sampling_weights
1504
+
1505
+ @property
1506
+ def trajectory_sampling_weights(self) -> list[np.ndarray]:
1507
+ """The sampling weights for each trajectory in each dataset."""
1508
+ return self._trajectory_sampling_weights
1509
+
1510
+ @property
1511
+ def primary_dataset_indices(self) -> np.ndarray:
1512
+ """The indices of the primary datasets."""
1513
+ return self._primary_dataset_indices
1514
+
1515
+ def __str__(self) -> str:
1516
+ dataset_descriptions = []
1517
+ for dataset, weight in zip(self.datasets, self.dataset_sampling_weights):
1518
+ dataset_description = {
1519
+ "Dataset": str(dataset),
1520
+ "Sampling weight": float(weight),
1521
+ }
1522
+ dataset_descriptions.append(dataset_description)
1523
+ return json.dumps({"Mixture dataset": dataset_descriptions}, indent=2)
1524
+
1525
+ def set_epoch(self, epoch: int):
1526
+ """Set the epoch for the dataset.
1527
+
1528
+ Args:
1529
+ epoch (int): The epoch to set.
1530
+ """
1531
+ self.epoch = epoch
1532
+ # self.sampled_steps = self.sample_epoch()
1533
+
1534
+ def sample_step(self, index: int) -> tuple[LeRobotSingleDataset, int, int]:
1535
+ """Sample a single step from the dataset."""
1536
+ # return self.sampled_steps[index]
1537
+
1538
+ # Set seed
1539
+ seed = index if self.mode != "train" else safe_hash((self.epoch, index, self.seed))
1540
+ rng = np.random.default_rng(seed)
1541
+
1542
+ # Sample dataset
1543
+ dataset_index = rng.choice(len(self.datasets), p=self.dataset_sampling_weights)
1544
+ dataset = self.datasets[dataset_index]
1545
+
1546
+ # Sample trajectory
1547
+ # trajectory_index = rng.choice(
1548
+ # len(dataset.trajectory_ids), p=self.trajectory_sampling_weights[dataset_index]
1549
+ # )
1550
+ # trajectory_id = dataset.trajectory_ids[trajectory_index]
1551
+
1552
+ # # Sample step
1553
+ # base_index = rng.choice(dataset.trajectory_lengths[trajectory_index])
1554
+ # return dataset, trajectory_id, base_index
1555
+ single_step_index = rng.choice(len(dataset.all_steps))
1556
+ trajectory_id, base_index = dataset.all_steps[single_step_index]
1557
+ return dataset, trajectory_id, base_index
1558
+
1559
+ def sample_step_with_shuffle_is_true(self, index: int) -> tuple[LeRobotSingleDataset, int, int]:
1560
+ """Sample a single step from the dataset."""
1561
+ '''only use this when DataLoader is initialized with shuffle=True'''
1562
+ print(index)
1563
+ # return self.sampled_steps[index]
1564
+
1565
+ # Set seed
1566
+ # seed = index if self.mode != "train" else safe_hash((self.epoch, index, self.seed))
1567
+ # rng = np.random.default_rng(seed)
1568
+
1569
+ # Sample dataset
1570
+ # dataset_index = rng.choice(len(self.datasets), p=self.dataset_sampling_weights)
1571
+ # dataset = self.datasets[dataset_index]
1572
+ dataset = self.datasets[0] #FIXME
1573
+
1574
+ # Sample trajectory
1575
+ # trajectory_index = rng.choice(
1576
+ # len(dataset.trajectory_ids), p=self.trajectory_sampling_weights[dataset_index]
1577
+ # )
1578
+ # trajectory_id = dataset.trajectory_ids[trajectory_index]
1579
+
1580
+ # # Sample step
1581
+ # base_index = rng.choice(dataset.trajectory_lengths[trajectory_index])
1582
+ # return dataset, trajectory_id, base_index
1583
+ # no random, follow the order of the dataset
1584
+ # single_step_index = rng.choice(len(dataset.all_steps))
1585
+ # trajectory_id, base_index = dataset.all_steps[single_step_index]
1586
+
1587
+ trajectory_id, base_index = dataset.all_steps[index]
1588
+ return dataset, trajectory_id, base_index
1589
+
1590
+ def __getitem__(self, index: int) -> dict:
1591
+ """Get the data for a single trajectory and start index.
1592
+
1593
+ Args:
1594
+ index (int): The index of the trajectory to get.
1595
+
1596
+ Returns:
1597
+ dict: The data for the trajectory and start index.
1598
+ """
1599
+ max_retries = 10
1600
+ last_exception = None
1601
+
1602
+ for attempt in range(max_retries):
1603
+ try:
1604
+ dataset, trajectory_name, step = self.sample_step(index)
1605
+ data = dataset.transforms(dataset.get_step_data(trajectory_name, step))
1606
+
1607
+ # Process all video keys dynamically
1608
+ images = []
1609
+ for video_key in dataset.modality_keys["video"]:
1610
+ image = data[video_key][0]
1611
+
1612
+ # Apply image cropping if enabled and the video key is base_view
1613
+ # Note: crop_obs_camera functionality has been removed
1614
+
1615
+ image = Image.fromarray(image).resize((224, 224))
1616
+ # # use time stamp as filename
1617
+ # import time
1618
+ # filename = f"/hfm/boqian/storage/tmp/image_{int(time.time())}_{index}.png"
1619
+ # image.save(filename)
1620
+ images.append(image)
1621
+
1622
+ # Get language and action data
1623
+ language = data[dataset.modality_keys["language"][0]][0]
1624
+ action = []
1625
+ for action_key in dataset.modality_keys["action"]:
1626
+ action.append(data[action_key])
1627
+ action = np.concatenate(action, axis=1).astype(np.float16)
1628
+
1629
+ return dict(action=action, image=images, lang=language)
1630
+
1631
+ except Exception as e:
1632
+ last_exception = e
1633
+ if attempt < max_retries - 1:
1634
+ # Log the error but continue trying
1635
+ print(f"Attempt {attempt + 1}/{max_retries} failed for index {index}: {e}")
1636
+ print(f"Retrying with new sample...")
1637
+ # For retry, we can use a slightly different index to get a new sample
1638
+ # This helps avoid getting stuck on the same problematic sample
1639
+ index = (index + 1) % len(self)
1640
+ else:
1641
+ # All retries exhausted
1642
+ print(f"All {max_retries} attempts failed for index {index}")
1643
+ print(f"Last error: {last_exception}")
1644
+ # Return a dummy sample or re-raise the exception
1645
+ raise last_exception
1646
+
1647
+ def __len__(self) -> int:
1648
+ """Get the length of a single epoch in the mixture.
1649
+
1650
+ Returns:
1651
+ int: The length of a single epoch in the mixture.
1652
+ """
1653
+ # Check for potential issues
1654
+ if len(self.datasets) == 0:
1655
+ return 0
1656
+
1657
+ # Check if any dataset lengths are 0 or NaN
1658
+ if np.any(self.dataset_lengths == 0) or np.any(np.isnan(self.dataset_lengths)):
1659
+ print(f"Warning: Found zero or NaN dataset lengths: {self.dataset_lengths}")
1660
+ # Filter out zero/NaN length datasets
1661
+ valid_indices = (self.dataset_lengths > 0) & (~np.isnan(self.dataset_lengths))
1662
+ if not np.any(valid_indices):
1663
+ print("Error: All datasets have zero or NaN length")
1664
+ return 0
1665
+ else:
1666
+ valid_indices = np.ones(len(self.datasets), dtype=bool)
1667
+
1668
+ # Check if any sampling weights are 0 or NaN
1669
+ if np.any(self.dataset_sampling_weights == 0) or np.any(np.isnan(self.dataset_sampling_weights)):
1670
+ print(f"Warning: Found zero or NaN sampling weights: {self.dataset_sampling_weights}")
1671
+ # Use only valid weights
1672
+ valid_weights = (self.dataset_sampling_weights > 0) & (~np.isnan(self.dataset_sampling_weights))
1673
+ valid_indices = valid_indices & valid_weights
1674
+ if not np.any(valid_indices):
1675
+ print("Error: All sampling weights are zero or NaN")
1676
+ return 0
1677
+
1678
+ # Check primary dataset indices
1679
+ primary_and_valid = self.primary_dataset_indices & valid_indices
1680
+ if not np.any(primary_and_valid):
1681
+ print(f"Warning: No valid primary datasets found. Primary indices: {self.primary_dataset_indices}, Valid indices: {valid_indices}")
1682
+ # Fallback: use the largest valid dataset
1683
+ if np.any(valid_indices):
1684
+ max_length = self.dataset_lengths[valid_indices].max()
1685
+ print(f"Fallback: Using maximum dataset length: {max_length}")
1686
+ return int(max_length)
1687
+ else:
1688
+ return 0
1689
+
1690
+ # Calculate the ratio and get max
1691
+ ratios = (self.dataset_lengths / self.dataset_sampling_weights)[primary_and_valid]
1692
+
1693
+ # Check for NaN or inf in ratios
1694
+ if np.any(np.isnan(ratios)) or np.any(np.isinf(ratios)):
1695
+ print(f"Warning: Found NaN or inf in ratios: {ratios}")
1696
+ print(f"Dataset lengths: {self.dataset_lengths[primary_and_valid]}")
1697
+ print(f"Sampling weights: {self.dataset_sampling_weights[primary_and_valid]}")
1698
+ # Filter out invalid ratios
1699
+ valid_ratios = ratios[~np.isnan(ratios) & ~np.isinf(ratios)]
1700
+ if len(valid_ratios) == 0:
1701
+ print("Error: All ratios are NaN or inf")
1702
+ return 0
1703
+ max_ratio = valid_ratios.max()
1704
+ else:
1705
+ max_ratio = ratios.max()
1706
+
1707
+ result = int(max_ratio)
1708
+ if result == 0:
1709
+ print(f"Warning: Dataset mixture length is 0")
1710
+ return result
1711
+
1712
+ @staticmethod
1713
+ def compute_overall_statistics(
1714
+ per_task_stats: list[dict[str, dict[str, list[float] | np.ndarray]]],
1715
+ dataset_sampling_weights: list[float] | np.ndarray,
1716
+ percentile_mixing_method: str = "weighted_average",
1717
+ ) -> dict[str, dict[str, list[float]]]:
1718
+ """
1719
+ Computes overall statistics from per-task statistics using dataset sample weights.
1720
+
1721
+ Args:
1722
+ per_task_stats: List of per-task statistics.
1723
+ Example format of one element in the per-task statistics list:
1724
+ {
1725
+ "state.gripper": {
1726
+ "min": [...],
1727
+ "max": [...],
1728
+ "mean": [...],
1729
+ "std": [...],
1730
+ "q01": [...],
1731
+ "q99": [...],
1732
+ },
1733
+ ...
1734
+ }
1735
+ dataset_sampling_weights: List of sample weights for each task.
1736
+ percentile_mixing_method: The method to mix the percentiles, either "weighted_average" or "weighted_std".
1737
+
1738
+ Returns:
1739
+ A dict of overall statistics per modality.
1740
+ """
1741
+ # Normalize the sample weights to sum to 1
1742
+ dataset_sampling_weights = np.array(dataset_sampling_weights)
1743
+ normalized_weights = dataset_sampling_weights / dataset_sampling_weights.sum()
1744
+
1745
+ # Initialize overall statistics dict
1746
+ overall_stats: dict[str, dict[str, list[float]]] = {}
1747
+
1748
+ # Get the list of modality keys
1749
+ modality_keys = per_task_stats[0].keys()
1750
+
1751
+ for modality in modality_keys:
1752
+ # Number of dimensions (assuming consistent across tasks)
1753
+ num_dims = len(per_task_stats[0][modality]["mean"])
1754
+
1755
+ # Initialize accumulators for means and variances
1756
+ weighted_means = np.zeros(num_dims)
1757
+ weighted_squares = np.zeros(num_dims)
1758
+
1759
+ # Collect min, max, q01, q99 from all tasks
1760
+ min_list = []
1761
+ max_list = []
1762
+ q01_list = []
1763
+ q99_list = []
1764
+
1765
+ for task_idx, task_stats in enumerate(per_task_stats):
1766
+ w_i = normalized_weights[task_idx]
1767
+ stats = task_stats[modality]
1768
+ means = np.array(stats["mean"])
1769
+ stds = np.array(stats["std"])
1770
+
1771
+ # Update weighted sums for mean and variance
1772
+ weighted_means += w_i * means
1773
+ weighted_squares += w_i * (stds**2 + means**2)
1774
+
1775
+ # Collect min, max, q01, q99
1776
+ min_list.append(stats["min"])
1777
+ max_list.append(stats["max"])
1778
+ q01_list.append(stats["q01"])
1779
+ q99_list.append(stats["q99"])
1780
+
1781
+ # Compute overall mean
1782
+ overall_mean = weighted_means.tolist()
1783
+
1784
+ # Compute overall variance and std deviation
1785
+ overall_variance = weighted_squares - weighted_means**2
1786
+ overall_std = np.sqrt(overall_variance).tolist()
1787
+
1788
+ # Compute overall min and max per dimension
1789
+ overall_min = np.min(np.array(min_list), axis=0).tolist()
1790
+ overall_max = np.max(np.array(max_list), axis=0).tolist()
1791
+
1792
+ # Compute overall q01 and q99 per dimension
1793
+ # Use weighted average of per-task quantiles
1794
+ q01_array = np.array(q01_list)
1795
+ q99_array = np.array(q99_list)
1796
+ if percentile_mixing_method == "weighted_average":
1797
+ weighted_q01 = np.average(q01_array, axis=0, weights=normalized_weights).tolist()
1798
+ weighted_q99 = np.average(q99_array, axis=0, weights=normalized_weights).tolist()
1799
+ # std_q01 = np.std(q01_array, axis=0).tolist()
1800
+ # std_q99 = np.std(q99_array, axis=0).tolist()
1801
+ # print(modality)
1802
+ # print(f"{std_q01=}, {std_q99=}")
1803
+ # print(f"{weighted_q01=}, {weighted_q99=}")
1804
+ elif percentile_mixing_method == "min_max":
1805
+ weighted_q01 = np.min(q01_array, axis=0).tolist()
1806
+ weighted_q99 = np.max(q99_array, axis=0).tolist()
1807
+ else:
1808
+ raise ValueError(f"Invalid percentile mixing method: {percentile_mixing_method}")
1809
+
1810
+ # Store the overall statistics for the modality
1811
+ overall_stats[modality] = {
1812
+ "min": overall_min,
1813
+ "max": overall_max,
1814
+ "mean": overall_mean,
1815
+ "std": overall_std,
1816
+ "q01": weighted_q01,
1817
+ "q99": weighted_q99,
1818
+ }
1819
+
1820
+ return overall_stats
1821
+
1822
+ @staticmethod
1823
+ def merge_metadata(
1824
+ metadatas: list[DatasetMetadata],
1825
+ dataset_sampling_weights: list[float],
1826
+ percentile_mixing_method: str,
1827
+ ) -> DatasetMetadata:
1828
+ """Merge multiple metadata into one."""
1829
+ # Convert to dicts
1830
+ metadata_dicts = [metadata.model_dump(mode="json") for metadata in metadatas]
1831
+ # Create a new metadata dict
1832
+ merged_metadata = {}
1833
+
1834
+ # Check all metadata have the same embodiment tag
1835
+ assert all(
1836
+ metadata.embodiment_tag == metadatas[0].embodiment_tag for metadata in metadatas
1837
+ ), "All metadata must have the same embodiment tag"
1838
+ merged_metadata["embodiment_tag"] = metadatas[0].embodiment_tag
1839
+
1840
+ # Merge the dataset statistics
1841
+ dataset_statistics = {}
1842
+ dataset_statistics["state"] = LeRobotMixtureDataset.compute_overall_statistics(
1843
+ per_task_stats=[m["statistics"]["state"] for m in metadata_dicts],
1844
+ dataset_sampling_weights=dataset_sampling_weights,
1845
+ percentile_mixing_method=percentile_mixing_method,
1846
+ )
1847
+ dataset_statistics["action"] = LeRobotMixtureDataset.compute_overall_statistics(
1848
+ per_task_stats=[m["statistics"]["action"] for m in metadata_dicts],
1849
+ dataset_sampling_weights=dataset_sampling_weights,
1850
+ percentile_mixing_method=percentile_mixing_method,
1851
+ )
1852
+ merged_metadata["statistics"] = dataset_statistics
1853
+
1854
+ # Merge the modality configs
1855
+ modality_configs = defaultdict(set)
1856
+ for metadata in metadata_dicts:
1857
+ for modality, configs in metadata["modalities"].items():
1858
+ modality_configs[modality].add(json.dumps(configs))
1859
+ merged_metadata["modalities"] = {}
1860
+ for modality, configs in modality_configs.items():
1861
+ # Check that all modality configs correspond to the same tag matches
1862
+ assert (
1863
+ len(configs) == 1
1864
+ ), f"Multiple modality configs for modality {modality}: {list(configs)}"
1865
+ merged_metadata["modalities"][modality] = json.loads(configs.pop())
1866
+
1867
+ return DatasetMetadata.model_validate(merged_metadata)
1868
+
1869
+ def update_metadata(self, metadata_config: dict, cached_statistics_path: Path | str | None = None) -> None:
1870
+ """
1871
+ Merge multiple metadatas into one and set the transforms with the merged metadata.
1872
+
1873
+ Args:
1874
+ metadata_config (dict): Configuration for the metadata.
1875
+ "percentile_mixing_method": The method to mix the percentiles, either "weighted_average" or "min_max".
1876
+ weighted_average: Use the weighted average of the percentiles using the weight used in sampling the datasets.
1877
+ min_max: Use the min of the 1st percentile and max of the 99th percentile.
1878
+ """
1879
+ # If cached path is provided, try to load and apply
1880
+ if cached_statistics_path is not None:
1881
+ try:
1882
+ cached_stats = self.load_merged_statistics(cached_statistics_path)
1883
+ self.apply_cached_statistics(cached_stats)
1884
+ return
1885
+ except (FileNotFoundError, KeyError, ValidationError) as e:
1886
+ print(f"Failed to load cached statistics: {e}")
1887
+ print("Falling back to computing statistics from scratch...")
1888
+
1889
+ self.tag = EmbodimentTag.NEW_EMBODIMENT.value
1890
+ self.merged_metadata: dict[str, DatasetMetadata] = {}
1891
+ # Group metadata by tag
1892
+ all_metadatas: dict[str, list[DatasetMetadata]] = {}
1893
+ for dataset in self.datasets:
1894
+ if dataset.tag not in all_metadatas:
1895
+ all_metadatas[dataset.tag] = []
1896
+ all_metadatas[dataset.tag].append(dataset.metadata)
1897
+ for tag, metadatas in all_metadatas.items():
1898
+ self.merged_metadata[tag] = self.merge_metadata(
1899
+ metadatas=metadatas,
1900
+ dataset_sampling_weights=self.dataset_sampling_weights.tolist(),
1901
+ percentile_mixing_method=metadata_config["percentile_mixing_method"],
1902
+ )
1903
+ for dataset in self.datasets:
1904
+ dataset.set_transforms_metadata(self.merged_metadata[dataset.tag])
1905
+
1906
+ def save_dataset_statistics(self, save_path: Path | str, format: str = "json") -> None:
1907
+ """
1908
+ Save merged dataset statistics to specified path in the required format.
1909
+ Only includes statistics for keys that are actually used in the datasets.
1910
+ Gripper-related keys will be placed at the end.
1911
+
1912
+ Args:
1913
+ save_path (Path | str): Path to save the statistics file
1914
+ format (str): Save format, currently only supports "json"
1915
+ """
1916
+ save_path = Path(save_path)
1917
+ save_path.parent.mkdir(parents=True, exist_ok=True)
1918
+
1919
+ # Build the data structure to save
1920
+ statistics_data = {}
1921
+
1922
+ # Collect actually used keys from all datasets
1923
+ all_used_action_keys = set()
1924
+ all_used_state_keys = set()
1925
+
1926
+ for dataset in self.datasets:
1927
+ used_action_keys, used_state_keys = get_used_modality_keys(dataset.modality_keys)
1928
+ all_used_action_keys.update(used_action_keys)
1929
+ all_used_state_keys.update(used_state_keys)
1930
+
1931
+ # Organize statistics by tag
1932
+ for tag, merged_metadata in self.merged_metadata.items():
1933
+ tag_stats = {}
1934
+
1935
+ # Process action statistics
1936
+ if hasattr(merged_metadata.statistics, 'action') and merged_metadata.statistics.action:
1937
+ action_stats = merged_metadata.statistics.action
1938
+
1939
+ # Filter and reorder keys
1940
+ non_gripper_keys = []
1941
+ gripper_keys = []
1942
+
1943
+ for key in action_stats.keys():
1944
+ if key in all_used_action_keys:
1945
+ if "gripper" in key.lower():
1946
+ gripper_keys.append(key)
1947
+ else:
1948
+ non_gripper_keys.append(key)
1949
+
1950
+ reordered_keys = non_gripper_keys + gripper_keys
1951
+
1952
+ filtered_action_stats = {}
1953
+ for key in reordered_keys:
1954
+ filtered_action_stats[key] = action_stats[key]
1955
+
1956
+ if filtered_action_stats:
1957
+ combined_action_stats = combine_modality_stats(filtered_action_stats)
1958
+
1959
+ mask = generate_action_mask_for_used_keys(
1960
+ merged_metadata.modalities.action, filtered_action_stats.keys()
1961
+ )
1962
+ combined_action_stats["mask"] = mask
1963
+
1964
+ tag_stats["action"] = combined_action_stats
1965
+
1966
+ # Process state statistics
1967
+ if hasattr(merged_metadata.statistics, 'state') and merged_metadata.statistics.state:
1968
+ state_stats = merged_metadata.statistics.state
1969
+
1970
+ # Filter and reorder keys
1971
+ non_gripper_keys = []
1972
+ gripper_keys = []
1973
+
1974
+ for key in state_stats.keys():
1975
+ if key in all_used_state_keys:
1976
+ if "gripper" in key.lower():
1977
+ gripper_keys.append(key)
1978
+ else:
1979
+ non_gripper_keys.append(key)
1980
+
1981
+ reordered_keys = non_gripper_keys + gripper_keys
1982
+
1983
+ filtered_state_stats = {}
1984
+ for key in reordered_keys:
1985
+ filtered_state_stats[key] = state_stats[key]
1986
+
1987
+ if filtered_state_stats:
1988
+ combined_state_stats = combine_modality_stats(filtered_state_stats)
1989
+ tag_stats["state"] = combined_state_stats
1990
+
1991
+ # Add dataset counts
1992
+ tag_stats.update(self._get_dataset_counts(tag))
1993
+
1994
+ statistics_data[tag] = tag_stats
1995
+
1996
+ # Save file
1997
+ if format.lower() == "json":
1998
+ if not str(save_path).endswith('.json'):
1999
+ save_path = save_path.with_suffix('.json')
2000
+ with open(save_path, 'w', encoding='utf-8') as f:
2001
+ json.dump(statistics_data, f, indent=2, ensure_ascii=False)
2002
+ else:
2003
+ raise ValueError(f"Unsupported format: {format}. Currently only 'json' is supported.")
2004
+
2005
+ print(f"Merged dataset statistics saved to: {save_path}")
2006
+ print(f"Used action keys (reordered): {list(all_used_action_keys)}")
2007
+ print(f"Used state keys (reordered): {list(all_used_state_keys)}")
2008
+
2009
+ def _combine_modality_stats(self, modality_stats: dict) -> dict:
2010
+ """Backward compatibility wrapper."""
2011
+ return combine_modality_stats(modality_stats)
2012
+
2013
+ def _generate_action_mask_for_used_keys(self, action_modalities: dict, used_action_keys_ordered) -> list[bool]:
2014
+ """Backward compatibility wrapper."""
2015
+ return generate_action_mask_for_used_keys(action_modalities, used_action_keys_ordered)
2016
+
2017
+ def _get_dataset_counts(self, tag: str) -> dict:
2018
+ """
2019
+ Get dataset count information for specified tag.
2020
+
2021
+ Args:
2022
+ tag (str): embodiment tag
2023
+
2024
+ Returns:
2025
+ dict: Dictionary containing num_transitions and num_trajectories
2026
+ """
2027
+ num_transitions = 0
2028
+ num_trajectories = 0
2029
+
2030
+ # Count dataset information belonging to this tag
2031
+ for dataset in self.datasets:
2032
+ if dataset.tag == tag:
2033
+ num_transitions += len(dataset)
2034
+ num_trajectories += len(dataset.trajectory_ids)
2035
+
2036
+ return {
2037
+ "num_transitions": num_transitions,
2038
+ "num_trajectories": num_trajectories
2039
+ }
2040
+
2041
+ @classmethod
2042
+ def load_merged_statistics(cls, load_path: Path | str) -> dict:
2043
+ """
2044
+ Load merged dataset statistics from file.
2045
+
2046
+ Args:
2047
+ load_path (Path | str): Path to the statistics file
2048
+
2049
+ Returns:
2050
+ dict: Dictionary containing merged statistics
2051
+ """
2052
+ load_path = Path(load_path)
2053
+ if not load_path.exists():
2054
+ raise FileNotFoundError(f"Statistics file not found: {load_path}")
2055
+
2056
+ if load_path.suffix.lower() == '.json':
2057
+ with open(load_path, 'r', encoding='utf-8') as f:
2058
+ return json.load(f)
2059
+ elif load_path.suffix.lower() == '.pkl':
2060
+ import pickle
2061
+ with open(load_path, 'rb') as f:
2062
+ return pickle.load(f)
2063
+ else:
2064
+ raise ValueError(f"Unsupported file format: {load_path.suffix}")
2065
+
2066
+ def apply_cached_statistics(self, cached_statistics: dict) -> None:
2067
+ """
2068
+ Apply cached statistics to avoid recomputation.
2069
+
2070
+ Args:
2071
+ cached_statistics (dict): Statistics loaded from file
2072
+ """
2073
+ # Validate that cached statistics match current datasets
2074
+ if "metadata" in cached_statistics:
2075
+ cached_dataset_names = set(cached_statistics["metadata"]["dataset_names"])
2076
+ current_dataset_names = set(dataset.dataset_name for dataset in self.datasets)
2077
+
2078
+ if cached_dataset_names != current_dataset_names:
2079
+ print("Warning: Cached statistics dataset names don't match current datasets.")
2080
+ print(f"Cached: {cached_dataset_names}")
2081
+ print(f"Current: {current_dataset_names}")
2082
+ return
2083
+
2084
+ # Apply cached statistics
2085
+ self.merged_metadata = {}
2086
+ for tag, stats_data in cached_statistics.items():
2087
+ if tag == "metadata": # Skip metadata field
2088
+ continue
2089
+
2090
+ # Convert back to DatasetMetadata format
2091
+ metadata_dict = {
2092
+ "embodiment_tag": tag,
2093
+ "statistics": {
2094
+ "action": {},
2095
+ "state": {}
2096
+ },
2097
+ "modalities": {}
2098
+ }
2099
+
2100
+ # Convert action statistics back
2101
+ if "action" in stats_data:
2102
+ action_data = stats_data["action"]
2103
+ # This is simplified - you may need to split back to sub-keys
2104
+ metadata_dict["statistics"]["action"] = action_data
2105
+
2106
+ # Convert state statistics back
2107
+ if "state" in stats_data:
2108
+ state_data = stats_data["state"]
2109
+ metadata_dict["statistics"]["state"] = state_data
2110
+
2111
+ self.merged_metadata[tag] = DatasetMetadata.model_validate(metadata_dict)
2112
+
2113
+ # Update transforms metadata for each dataset
2114
+ for dataset in self.datasets:
2115
+ if dataset.tag in self.merged_metadata:
2116
+ dataset.set_transforms_metadata(self.merged_metadata[dataset.tag])
2117
+
2118
+ print(f"Applied cached statistics for {len(self.merged_metadata)} embodiment tags.")