diff --git a/nix/lib/runtime.nix b/nix/lib/runtime.nix new file mode 100644 index 0000000000000000000000000000000000000000..fba4cb6c645cd9ebed5e4573c9d581576cbba786 --- /dev/null +++ b/nix/lib/runtime.nix @@ -0,0 +1,74 @@ +{ lib, pkgs }: + +let + exportEnv = env: + lib.concatStringsSep "\n" ( + lib.mapAttrsToList (name: value: "export ${name}=${lib.escapeShellArg value}") env + ); + + prependEnv = env: + lib.concatStringsSep "\n" ( + lib.mapAttrsToList ( + name: value: + '' + _runtime_current_value="$(printenv ${name} || true)" + if [ -n "$_runtime_current_value" ]; then + export ${name}=${lib.escapeShellArg value}:"$_runtime_current_value" + else + export ${name}=${lib.escapeShellArg value} + fi + unset _runtime_current_value + '' + ) env + ); + + mergeStringAttrs = runtimes: key: + lib.foldl' ( + acc: runtime: + let current = runtime.${key} or { }; + in acc // lib.mapAttrs ( + name: value: + if builtins.hasAttr name acc + then "${value}:${builtins.getAttr name acc}" + else value + ) current + ) { } runtimes; + + mergeRuntimes = runtimes: { + packages = lib.unique (lib.concatMap (runtime: runtime.packages or [ ]) runtimes); + env = lib.foldl' (acc: runtime: acc // (runtime.env or { })) { } runtimes; + prependEnv = mergeStringAttrs runtimes "prependEnv"; + shellHook = lib.concatStringsSep "\n\n" ( + lib.filter (chunk: chunk != "") (map (runtime: runtime.shellHook or "") runtimes) + ); + }; + + mkRuntimeShell = { + name, + runtimes, + extraPackages ? [ ], + extraShellHook ? "", + stdenv ? null, + }: + let + runtime = mergeRuntimes runtimes; + shellHook = lib.concatStringsSep "\n\n" ( + [ + (exportEnv (runtime.env or { })) + (prependEnv (runtime.prependEnv or { })) + (runtime.shellHook or "") + extraShellHook + '' + echo "${name} shell ready" + '' + ] + ); + in + pkgs.mkShell ({ + packages = lib.unique (extraPackages ++ runtime.packages); + inherit shellHook; + } // lib.optionalAttrs (stdenv != null) { inherit stdenv; }); +in +{ + inherit exportEnv prependEnv mergeRuntimes mkRuntimeShell; +} diff --git a/nix/runtime-base.nix b/nix/runtime-base.nix new file mode 100644 index 0000000000000000000000000000000000000000..e0c56606fe7e1ce8b6263341a13357d1fc26c5ed --- /dev/null +++ b/nix/runtime-base.nix @@ -0,0 +1,49 @@ +{ pkgs }: + +let + cuda = pkgs.cudaPackages_12_8.cudatoolkit; + gcc = pkgs.gcc13; + ffmpeg = pkgs.ffmpeg_7; + ffmpegLib = ffmpeg.lib; + caBundle = "${pkgs.cacert}/etc/ssl/certs/ca-bundle.crt"; +in +{ + packages = [ + pkgs.cmake + pkgs.ninja + pkgs.pkg-config + gcc + pkgs.cacert + pkgs.linuxHeaders + ffmpeg + cuda + ]; + + env = { + CUDA_HOME = "${cuda}"; + CUDA_PATH = "${cuda}"; + CUDACXX = "${cuda}/bin/nvcc"; + CUDAHOSTCXX = "${gcc}/bin/g++"; + CC = "${gcc}/bin/gcc"; + CXX = "${gcc}/bin/g++"; + CMAKE_CUDA_HOST_COMPILER = "${gcc}/bin/gcc"; + NVCC_PREPEND_FLAGS = "-ccbin ${gcc}/bin/gcc"; + SSL_CERT_FILE = caBundle; + NIX_SSL_CERT_FILE = caBundle; + REQUESTS_CA_BUNDLE = caBundle; + CURL_CA_BUNDLE = caBundle; + GIT_SSL_CAINFO = caBundle; + }; + + prependEnv = { + PATH = "${ffmpeg}/bin"; + CPATH = "${pkgs.linuxHeaders}/include:${cuda}/include"; + LIBRARY_PATH = "${cuda}/lib64:${cuda}/lib"; + LD_LIBRARY_PATH = "${ffmpegLib}/lib:${cuda}/lib64:${cuda}/lib:${pkgs.stdenv.cc.cc.lib}/lib"; + }; + + shellHook = '' + export CFLAGS="-I${pkgs.linuxHeaders}/include ''${CFLAGS:-}" + export CXXFLAGS="-I${pkgs.linuxHeaders}/include ''${CXXFLAGS:-}" + ''; +} diff --git a/nix/runtime-host-gpu.nix b/nix/runtime-host-gpu.nix new file mode 100644 index 0000000000000000000000000000000000000000..bd2486488e9581c905891989706de07cee1588af --- /dev/null +++ b/nix/runtime-host-gpu.nix @@ -0,0 +1,101 @@ +{ pkgs }: + +{ + shellHook = '' + _runtime_detect_and_prepend() { + _runtime_var_name="$1" + shift + _runtime_joined_paths="" + for _runtime_path in "$@"; do + if [ -e "$_runtime_path" ]; then + if [ -n "$_runtime_joined_paths" ]; then + _runtime_joined_paths="$_runtime_joined_paths:$_runtime_path" + else + _runtime_joined_paths="$_runtime_path" + fi + fi + done + + if [ -n "$_runtime_joined_paths" ]; then + eval "_runtime_current_value=\''${$_runtime_var_name}" + if [ -n "$_runtime_current_value" ]; then + export "$_runtime_var_name=$_runtime_joined_paths:$_runtime_current_value" + else + export "$_runtime_var_name=$_runtime_joined_paths" + fi + unset _runtime_current_value + fi + + unset _runtime_joined_paths + unset _runtime_var_name + unset _runtime_path + } + + if [ -z "''${VK_ICD_FILENAMES:-}" ]; then + for _runtime_vk_icd in \ + /run/opengl-driver/share/vulkan/icd.d/nvidia_icd.x86_64.json \ + /usr/share/vulkan/icd.d/nvidia_icd.json \ + /usr/share/vulkan/icd.d/nvidia_icd.x86_64.json \ + /etc/vulkan/icd.d/nvidia_icd.json + do + if [ -f "$_runtime_vk_icd" ]; then + export VK_ICD_FILENAMES="$_runtime_vk_icd" + break + fi + done + unset _runtime_vk_icd + fi + + if [ -z "''${__EGL_VENDOR_LIBRARY_FILENAMES:-}" ]; then + for _runtime_egl_vendor in \ + /run/opengl-driver/share/glvnd/egl_vendor.d/10_nvidia.json \ + /usr/share/glvnd/egl_vendor.d/10_nvidia.json \ + /usr/share/glvnd/egl_vendor.d/50_nvidia.json + do + if [ -f "$_runtime_egl_vendor" ]; then + export __EGL_VENDOR_LIBRARY_FILENAMES="$_runtime_egl_vendor" + break + fi + done + unset _runtime_egl_vendor + fi + + _runtime_detect_and_prepend LD_LIBRARY_PATH \ + /run/opengl-driver/lib \ + /run/opengl-driver-32/lib \ + /usr/lib/x86_64-linux-gnu/nvidia/current \ + /usr/lib/x86_64-linux-gnu/nvidia \ + /usr/lib/nvidia \ + /usr/lib/nvidia-570 \ + /usr/lib/nvidia-575 \ + /usr/lib/nvidia-580 \ + /usr/lib/nvidia-590 \ + /usr/lib/wsl/lib + + _runtime_detect_and_prepend XDG_DATA_DIRS \ + /run/opengl-driver/share \ + /usr/share + + if [ -z "''${TRITON_LIBCUDA_PATH:-}" ]; then + for _runtime_triton_cuda in \ + /run/opengl-driver/lib \ + /usr/lib/x86_64-linux-gnu \ + /usr/lib/x86_64-linux-gnu/nvidia/current \ + /usr/lib/x86_64-linux-gnu/nvidia \ + /usr/lib/nvidia-590 \ + /usr/lib/nvidia-580 \ + /usr/lib/nvidia-575 \ + /usr/lib/nvidia-570 \ + /usr/lib/wsl/lib + do + if [ -d "$_runtime_triton_cuda" ]; then + export TRITON_LIBCUDA_PATH="$_runtime_triton_cuda" + break + fi + done + unset _runtime_triton_cuda + fi + + unset -f _runtime_detect_and_prepend + ''; +} diff --git a/nix/runtime.nix b/nix/runtime.nix new file mode 100644 index 0000000000000000000000000000000000000000..2d95ad68a71c5e16194bf68e9fd707135e7aebe4 --- /dev/null +++ b/nix/runtime.nix @@ -0,0 +1,14 @@ +{ pkgs }: + +let + runtimeLib = import ./lib/runtime.nix { + lib = pkgs.lib; + inherit pkgs; + }; + baseRuntime = import ./runtime-base.nix { inherit pkgs; }; + hostGpuRuntime = import ./runtime-host-gpu.nix { inherit pkgs; }; +in +runtimeLib.mergeRuntimes [ + baseRuntime + hostGpuRuntime +] diff --git a/scripts/deepspeed/zero2.json b/scripts/deepspeed/zero2.json new file mode 100644 index 0000000000000000000000000000000000000000..c95ebefe07b7d8d9fd0936a014679d07102cc270 --- /dev/null +++ b/scripts/deepspeed/zero2.json @@ -0,0 +1,23 @@ +{ + "fp16": { + "enabled": "auto", + "loss_scale": 0, + "loss_scale_window": 1000, + "initial_scale_power": 16, + "hysteresis": 2, + "min_loss_scale": 1 + }, + "bf16": { + "enabled": "auto" + }, + "train_micro_batch_size_per_gpu": "auto", + "train_batch_size": "auto", + "gradient_accumulation_steps": "auto", + "zero_optimization": { + "stage": 2, + "overlap_comm": true, + "contiguous_gradients": true, + "sub_group_size": 1e9, + "reduce_bucket_size": "auto" + } +} \ No newline at end of file diff --git a/scripts/deepspeed/zero3.json b/scripts/deepspeed/zero3.json new file mode 100644 index 0000000000000000000000000000000000000000..9e89bf2672cb75a8e1d3f51a7914d347e6add2aa --- /dev/null +++ b/scripts/deepspeed/zero3.json @@ -0,0 +1,29 @@ +{ + "fp16": { + "enabled": "auto", + "loss_scale": 0, + "loss_scale_window": 1000, + "initial_scale_power": 16, + "hysteresis": 2, + "min_loss_scale": 1 + }, + "bf16": { + "enabled": "auto" + }, + "train_micro_batch_size_per_gpu": "auto", + "train_batch_size": "auto", + "gradient_accumulation_steps": "auto", + "gradient_clipping": 1.0, + "zero_optimization": { + "stage": 3, + "overlap_comm": true, + "contiguous_gradients": true, + "sub_group_size": 1e9, + "reduce_bucket_size": "auto", + "stage3_prefetch_bucket_size": "auto", + "stage3_param_persistence_threshold": "auto", + "stage3_max_live_parameters": 1e9, + "stage3_max_reuse_distance": 1e9, + "stage3_gather_16bit_weights_on_model_save": true + } +} \ No newline at end of file diff --git a/scripts/deepspeed/zero3_offload.json b/scripts/deepspeed/zero3_offload.json new file mode 100644 index 0000000000000000000000000000000000000000..e0a54c2c2bc10f76458c42a43de0970a9251759f --- /dev/null +++ b/scripts/deepspeed/zero3_offload.json @@ -0,0 +1,56 @@ +{ + "fp16": { + "enabled": "auto", + "loss_scale": 0, + "loss_scale_window": 1000, + "initial_scale_power": 16, + "hysteresis": 2, + "min_loss_scale": 1 + }, + "bf16": { + "enabled": "auto" + }, + "optimizer": { + "type": "AdamW", + "params": { + "lr": "auto", + "betas": "auto", + "eps": "auto", + "weight_decay": "auto" + } + }, + "scheduler": { + "type": "WarmupLR", + "params": { + "warmup_min_lr": "auto", + "warmup_max_lr": "auto", + "warmup_num_steps": "auto" + } + }, + "zero_optimization": { + "stage": 3, + "offload_optimizer": { + "device": "cpu", + "pin_memory": true + }, + "offload_param": { + "device": "cpu", + "pin_memory": true + }, + "overlap_comm": true, + "contiguous_gradients": true, + "sub_group_size": 1e9, + "reduce_bucket_size": "auto", + "stage3_prefetch_bucket_size": "auto", + "stage3_param_persistence_threshold": "auto", + "stage3_max_live_parameters": 1e9, + "stage3_max_reuse_distance": 1e9, + "gather_16bit_weights_on_model_save": true + }, + "gradient_accumulation_steps": "auto", + "gradient_clipping": "auto", + "train_batch_size": "auto", + "train_micro_batch_size_per_gpu": "auto", + "steps_per_print": 1e5, + "wall_clock_breakdown": false +} \ No newline at end of file diff --git a/scripts/infer/multiturn_auto_scenario_backchannel_train_v1.json b/scripts/infer/multiturn_auto_scenario_backchannel_train_v1.json new file mode 100644 index 0000000000000000000000000000000000000000..1940b3b5c93d1789d80d7e08a0aec7171417ee1e --- /dev/null +++ b/scripts/infer/multiturn_auto_scenario_backchannel_train_v1.json @@ -0,0 +1,48 @@ +{ + "format": "hfd_multiturn_auto_infer_scenario_v1", + "name": "backchannel_grab_mmhmm_wave_zh", + "interaction_mode": "backchannel", + "data_split": "train", + "split_seed": 42, + "val_ratio": 0.1, + "split_source_namespace": "seed", + "description": "第一条动作执行中插入同音色的训练集短反馈“嗯嗯”,动作必须继续;随后再给一条真正的新指令。", + "required_semantics_version": 22, + "initial_idle_steps": 5, + "initial_idle_seconds": 2.0, + "motion_frames_per_turn": 450, + "motion_seconds_per_turn": 9.0, + "inter_turn_background_steps_by_turn": [10, 15], + "inter_turn_background_seconds_by_turn": [4.0, 6.0], + "terminal_background_steps": 5, + "terminal_background_seconds": 2.0, + "turns": [ + { + "turn_index": 1, + "source_motion_filename": "grab_walk_ff_180_001__A548", + "audio_path": "/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/qwen3-tts/seed_audio/000006/female_01_zh.wav", + "spoken_command": "向前走并抓取某物", + "ideal_motion": "听完后向前走并抓取,动作持续执行。" + }, + { + "turn_index": 2, + "backchannel_intent": "continuer", + "audio_path": "/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/videogen/hfd_backchannel_commands_v2_short/audio/train/bc_cont_01/female_01_zh.wav", + "spoken_command": "嗯嗯", + "ideal_motion": "这是短反馈而不是新指令;控制持续输出 ,抓取动作的相位、RTC 和 future progress 均连续,不进入 listen 或 idle。" + }, + { + "turn_index": 3, + "source_motion_filename": "wave_two_hands_R_001__A533", + "audio_path": "/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/qwen3-tts/seed_audio/000155/female_01_zh.wav", + "spoken_command": "双手在身前挥动", + "ideal_motion": "这是真正的新指令,模型才应进入 listen,随后从当前状态执行双手挥动。" + } + ], + "ideal_control_timeline": [ + "第一条动作指令遵循 idle→listen→motion", + "动作中听到“嗯嗯”仍保持 motion,不重置动作相位", + "后续真正的新指令到来后才从 motion 切到 listen,再进入新 motion", + "末尾纯背景回到 idle" + ] +} diff --git a/scripts/infer/multiturn_auto_scenario_backchannel_val_v1.json b/scripts/infer/multiturn_auto_scenario_backchannel_val_v1.json new file mode 100644 index 0000000000000000000000000000000000000000..b5c718ffe5ebbcdecf9779116ad2793d8212d0e1 --- /dev/null +++ b/scripts/infer/multiturn_auto_scenario_backchannel_val_v1.json @@ -0,0 +1,48 @@ +{ + "format": "hfd_multiturn_auto_infer_scenario_v1", + "name": "backchannel_jog_indeed_pick_en", + "interaction_mode": "backchannel", + "data_split": "val", + "split_seed": 42, + "val_ratio": 0.1, + "split_source_namespace": "seed", + "description": "未见过的验证动作与验证短反馈组合;检查 backchannel 的语义泛化,而不是记住固定词。", + "required_semantics_version": 22, + "initial_idle_steps": 4, + "initial_idle_seconds": 1.6, + "motion_frames_per_turn": 450, + "motion_seconds_per_turn": 9.0, + "inter_turn_background_steps_by_turn": [9, 16], + "inter_turn_background_seconds_by_turn": [3.6, 6.4], + "terminal_background_steps": 6, + "terminal_background_seconds": 2.4, + "turns": [ + { + "turn_index": 1, + "source_motion_filename": "jog_ff_start_180_R_003__A068", + "audio_path": "/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/qwen3-tts/seed_audio/000175/female_02_en.wav", + "spoken_command": "start jog forward", + "ideal_motion": "听完后向前慢跑,并保持连续闭环状态。" + }, + { + "turn_index": 2, + "backchannel_intent": "continuer", + "audio_path": "/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/videogen/hfd_backchannel_commands_v2_short/audio/val/bc_cont_11/female_02_en.wav", + "spoken_command": "Indeed.", + "ideal_motion": "验证集短反馈不应抢占控制权;慢跑动作、RTC、进度和 Qwen KV 均连续。" + }, + { + "turn_index": 3, + "source_motion_filename": "medium_big_light_two_hands_right_side_low_to_right_side_high_R_001__A529", + "audio_path": "/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/qwen3-tts/seed_audio/000227/female_02_en.wav", + "spoken_command": "picks up medium big light object with two hands from low right side and puts it away to the right side high", + "ideal_motion": "真正的新指令到来后才切入 listen,随后平滑执行右侧双手取放动作。" + } + ], + "ideal_control_timeline": [ + "初始背景输出 idle,第一条动作指令进入 listen 后执行 motion", + "未见过的短反馈期间持续输出 motion,不出现假 interruption", + "下一条真实指令出现后才切换 listen 和新 motion", + "最后在背景中回到 idle" + ] +} diff --git a/scripts/infer/multiturn_auto_scenario_emergency_stop_train_v1.json b/scripts/infer/multiturn_auto_scenario_emergency_stop_train_v1.json new file mode 100644 index 0000000000000000000000000000000000000000..a252fc374cf4c5477e5fe6b2b2458ed22e5e063e --- /dev/null +++ b/scripts/infer/multiturn_auto_scenario_emergency_stop_train_v1.json @@ -0,0 +1,48 @@ +{ + "format": "hfd_multiturn_auto_infer_scenario_v1", + "name": "stop_train_grab_stop_wave_zh", + "interaction_mode": "emergency-stop", + "data_split": "train", + "split_seed": 42, + "val_ratio": 0.1, + "split_source_namespace": "seed", + "description": "训练集抓取行走执行中收到中文急停,停住并保持两秒背景后再接挥手指令,检查控制后的恢复交互。", + "required_semantics_version": 21, + "initial_idle_steps": 5, + "initial_idle_seconds": 2.0, + "motion_frames_per_turn": 450, + "motion_seconds_per_turn": 9.0, + "inter_turn_background_steps_by_turn": [8, 5], + "inter_turn_background_seconds_by_turn": [3.2, 2.0], + "terminal_background_steps": 5, + "terminal_background_seconds": 2.0, + "turns": [ + { + "turn_index": 1, + "source_motion_filename": "grab_walk_ff_180_001__A548", + "audio_path": "/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/qwen3-tts/seed_audio/000006/female_01_zh.wav", + "spoken_command": "向前走并抓取某物", + "ideal_motion": "听完后向前行走并开始抓取。" + }, + { + "turn_index": 2, + "control_intent": "emergency-stop", + "audio_path": "/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/videogen/hfd_control_commands_v1/audio/emergency-stop/stop_02/female_01_zh.wav", + "spoken_command": "马上停下来", + "ideal_motion": "完整听到急停后直接从 motion 切到 idle,平滑停住,不出现 listen。" + }, + { + "turn_index": 3, + "source_motion_filename": "wave_two_hands_R_001__A533", + "audio_path": "/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/qwen3-tts/seed_audio/000155/female_01_zh.wav", + "spoken_command": "双手在身前挥动", + "ideal_motion": "急停后保持约两秒,再正常进入 listen 并执行双手挥动。" + } + ], + "ideal_control_timeline": [ + "初始背景为 ,第一条动作指令期间为 ", + "执行 约 3.2 秒后收到急停语音", + "急停语音未听完前仍是 ,听完后直接转 ", + "保持约两秒连续背景后接收新指令,重新进入 → " + ] +} diff --git a/scripts/infer/multiturn_auto_scenario_emergency_stop_val_v1.json b/scripts/infer/multiturn_auto_scenario_emergency_stop_val_v1.json new file mode 100644 index 0000000000000000000000000000000000000000..b0645e14811ae1ba711c59612eaeae81adf056ac --- /dev/null +++ b/scripts/infer/multiturn_auto_scenario_emergency_stop_val_v1.json @@ -0,0 +1,40 @@ +{ + "format": "hfd_multiturn_auto_infer_scenario_v1", + "name": "stop_val_jog_en", + "interaction_mode": "emergency-stop", + "data_split": "val", + "split_seed": 42, + "val_ratio": 0.1, + "split_source_namespace": "seed", + "description": "未见过的慢跑 motion 执行中收到英文急停,检查特殊指令的泛化。", + "required_semantics_version": 21, + "initial_idle_steps": 4, + "initial_idle_seconds": 1.6, + "motion_frames_per_turn": 450, + "motion_seconds_per_turn": 9.0, + "inter_turn_background_steps_by_turn": [9], + "inter_turn_background_seconds_by_turn": [3.6], + "terminal_background_steps": 6, + "terminal_background_seconds": 2.4, + "turns": [ + { + "turn_index": 1, + "source_motion_filename": "jog_ff_start_180_R_003__A068", + "audio_path": "/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/qwen3-tts/seed_audio/000175/female_02_en.wav", + "spoken_command": "start jog forward", + "ideal_motion": "开始向前慢跑。" + }, + { + "turn_index": 2, + "control_intent": "emergency-stop", + "audio_path": "/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/videogen/hfd_control_commands_v1/audio/emergency-stop/stop_02/female_02_en.wav", + "spoken_command": "Stop now.", + "ideal_motion": "完整听到 stop 后直接平滑停住,不进入 listen。" + } + ], + "ideal_control_timeline": [ + "idle -> listen -> motion", + "急停语音在 motion 中到来", + "听清后 motion -> idle,中间没有 listen" + ] +} diff --git a/scripts/infer/multiturn_auto_scenario_hesitation_filled_train_v2.json b/scripts/infer/multiturn_auto_scenario_hesitation_filled_train_v2.json new file mode 100644 index 0000000000000000000000000000000000000000..02bdfa5af5ec4d9ca582ace28b82ee0d0e013ea1 --- /dev/null +++ b/scripts/infer/multiturn_auto_scenario_hesitation_filled_train_v2.json @@ -0,0 +1,60 @@ +{ + "format": "hfd_multiturn_auto_infer_scenario_v1", + "name": "filled_wave_then_left_zh", + "interaction_mode": "hesitation", + "interaction_subtype": "filled-pause", + "data_split": "train", + "split_seed": 42, + "val_ratio": 0.1, + "split_source_namespace": "seed", + "description": "训练集中文指令中插入同音色、自然拖长的“呃……”,实际不确定区间为 2.4 秒,最终意图不变。", + "required_semantics_version": 24, + "initial_idle_steps": 5, + "initial_idle_seconds": 2.0, + "motion_frames_per_turn": 450, + "motion_seconds_per_turn": 9.0, + "inter_turn_background_steps_by_turn": [ + 25 + ], + "inter_turn_background_seconds_by_turn": [ + 10.0 + ], + "terminal_background_steps": 5, + "terminal_background_seconds": 2.0, + "disfluency_event": { + "turn_index": 0, + "subtype": "filled-pause", + "gap_steps": 6, + "gap_seconds": 2.4, + "marker_id": "hes_fill_zh_02" + }, + "turns": [ + { + "turn_index": 1, + "disfluency_subtype": "filled-pause", + "source_motion_filename": "wave_two_hands_R_001__A533", + "audio_path": "/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/videogen/hfd_hesitation_pause_v2/audio/review_examples/train/filled_wave_zh_female_01_zh.wav", + "spoken_command": "双手在身前……呃……挥动", + "final_instruction": "双手在身前挥动", + "ideal_motion": "听到呃时继续 listen,不提前抬手;完整指令结束后再完成双手挥动。" + }, + { + "turn_index": 2, + "source_motion_filename": "neutral_walk_ff_270_R_001__A534_M", + "audio_path": "/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/qwen3-tts/seed_audio/000120/female_01_zh.wav", + "spoken_command": "向左走路", + "ideal_motion": "随后相对当前朝向向左走。" + } + ], + "ideal_control_timeline": [ + "背景 idle", + "前半句、呃和后半句均为 listen", + "只有最终意图完整后才进入 motion", + "第二轮正常执行" + ], + "continuity_requirements": [ + "同一女性音色和语言", + "同一 Qwen KV 与连续背景", + "RTC 和动作相位不在犹豫处重置" + ] +} diff --git a/scripts/infer/multiturn_auto_scenario_hesitation_filled_val_v2.json b/scripts/infer/multiturn_auto_scenario_hesitation_filled_val_v2.json new file mode 100644 index 0000000000000000000000000000000000000000..112ccacce854e97940e2fc7774ea612af29b739f --- /dev/null +++ b/scripts/infer/multiturn_auto_scenario_hesitation_filled_val_v2.json @@ -0,0 +1,60 @@ +{ + "format": "hfd_multiturn_auto_infer_scenario_v1", + "name": "filled_pick_then_jog_en", + "interaction_mode": "hesitation", + "interaction_subtype": "filled-pause", + "data_split": "val", + "split_seed": 42, + "val_ratio": 0.1, + "split_source_namespace": "seed", + "description": "验证集英文长指令中插入未用于训练的同音色、自然拖长的 Hmm,检查长语义在 KV 中继续累积。", + "required_semantics_version": 24, + "initial_idle_steps": 4, + "initial_idle_seconds": 1.6, + "motion_frames_per_turn": 450, + "motion_seconds_per_turn": 9.0, + "inter_turn_background_steps_by_turn": [ + 25 + ], + "inter_turn_background_seconds_by_turn": [ + 10.0 + ], + "terminal_background_steps": 6, + "terminal_background_seconds": 2.4, + "disfluency_event": { + "turn_index": 0, + "subtype": "filled-pause", + "gap_steps": 5, + "gap_seconds": 2.0, + "marker_id": "hes_fill_en_11" + }, + "turns": [ + { + "turn_index": 1, + "disfluency_subtype": "filled-pause", + "source_motion_filename": "medium_big_light_two_hands_right_side_low_to_right_side_high_R_001__A529", + "audio_path": "/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/videogen/hfd_hesitation_pause_v2/audio/review_examples/val/filled_pick_en_female_02_en.wav", + "spoken_command": "picks up ... from low ... Hmm ... right side and puts it away to the right side high", + "final_instruction": "picks up medium big light object with two hands from low right side and puts it away to the right side high", + "ideal_motion": "Hmm 前后都保持 listen,完整结束后才执行右下到右上的双手取放。" + }, + { + "turn_index": 2, + "source_motion_filename": "jog_ff_start_180_R_003__A068", + "audio_path": "/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/qwen3-tts/seed_audio/000175/female_02_en.wav", + "spoken_command": "start jog forward", + "ideal_motion": "随后从当前状态向前慢跑。" + } + ], + "ideal_control_timeline": [ + "初始 idle", + "整条含 Hmm 的未完成指令保持 listen", + "完整语义后进入 motion", + "第二轮正常执行" + ], + "continuity_requirements": [ + "验证 marker 与训练 marker 含义拆分", + "同一音色、KV、背景和 RTC", + "两条 motion source 均属于 val" + ] +} diff --git a/scripts/infer/multiturn_auto_scenario_hesitation_repair_train_v1.json b/scripts/infer/multiturn_auto_scenario_hesitation_repair_train_v1.json new file mode 100644 index 0000000000000000000000000000000000000000..0bd2e7baa0060084bf3c5b49300a510c4df51c20 --- /dev/null +++ b/scripts/infer/multiturn_auto_scenario_hesitation_repair_train_v1.json @@ -0,0 +1,42 @@ +{ + "format": "hfd_multiturn_auto_infer_scenario_v1", + "name": "repair_grab_to_left_then_wave_zh", + "interaction_mode": "hesitation", + "interaction_subtype": "self-repair", + "data_split": "train", + "split_seed": 42, + "val_ratio": 0.1, + "split_source_namespace": "seed", + "description": "先说出抓取指令的不完整前缀,再用同音色明确改口为完整向左走;只能执行最终指令。", + "required_semantics_version": 23, + "initial_idle_steps": 5, + "initial_idle_seconds": 2.0, + "motion_frames_per_turn": 450, + "motion_seconds_per_turn": 9.0, + "inter_turn_background_steps_by_turn": [25], + "inter_turn_background_seconds_by_turn": [10.0], + "terminal_background_steps": 5, + "terminal_background_seconds": 2.0, + "disfluency_event": {"turn_index": 0, "subtype": "self-repair", "gap_steps": 7, "gap_seconds": 2.8, "marker_id": "repair_zh_01"}, + "turns": [ + { + "turn_index": 1, + "disfluency_subtype": "self-repair", + "source_motion_filename": "neutral_walk_ff_270_R_001__A534_M", + "audio_path": "/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/videogen/hfd_hesitation_pause_v1/audio/review_examples/train/repair_grab_to_left_zh_female_01_zh.wav", + "spoken_command": "向前……不对,我是说……向左走路", + "provisional_instruction": "向前走并抓取某物", + "final_instruction": "向左走路", + "ideal_motion": "整个改口过程只 listen;不得向前抓取,最终只相对当前朝向向左走。" + }, + { + "turn_index": 2, + "source_motion_filename": "wave_two_hands_R_001__A533", + "audio_path": "/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/qwen3-tts/seed_audio/000155/female_01_zh.wav", + "spoken_command": "双手在身前挥动", + "ideal_motion": "第二轮正常挥手,验证最终意图替换没有破坏后续交互。" + } + ], + "ideal_control_timeline": ["初始 idle", "错误前缀、改口词和完整新指令全程 listen", "只在完整最终指令后进入 motion", "不为错误前缀启动 motion"], + "continuity_requirements": ["旧意图与最终意图共用同一 Qwen KV,不靠清缓存作弊", "同一音色和背景", "动作与 RTC 只从最终意图开始"] +} diff --git a/scripts/infer/multiturn_auto_scenario_hesitation_repair_train_v2.json b/scripts/infer/multiturn_auto_scenario_hesitation_repair_train_v2.json new file mode 100644 index 0000000000000000000000000000000000000000..f9c04d7ba012b48e22a00c097491a5d2e18b8d00 --- /dev/null +++ b/scripts/infer/multiturn_auto_scenario_hesitation_repair_train_v2.json @@ -0,0 +1,63 @@ +{ + "format": "hfd_multiturn_auto_infer_scenario_v1", + "name": "repair_grab_to_left_then_wave_zh", + "interaction_mode": "hesitation", + "interaction_subtype": "self-repair", + "data_split": "train", + "split_seed": 42, + "val_ratio": 0.1, + "split_source_namespace": "seed", + "description": "先说出抓取指令的不完整前缀,再用同音色明确改口为完整向左走;只能执行最终指令。", + "required_semantics_version": 24, + "initial_idle_steps": 5, + "initial_idle_seconds": 2.0, + "motion_frames_per_turn": 450, + "motion_seconds_per_turn": 9.0, + "inter_turn_background_steps_by_turn": [ + 25 + ], + "inter_turn_background_seconds_by_turn": [ + 10.0 + ], + "terminal_background_steps": 5, + "terminal_background_seconds": 2.0, + "disfluency_event": { + "turn_index": 0, + "subtype": "self-repair", + "gap_steps": 7, + "gap_seconds": 2.8, + "marker_id": "repair_zh_01", + "repair_style": "explicit-cue" + }, + "turns": [ + { + "turn_index": 1, + "disfluency_subtype": "self-repair", + "source_motion_filename": "neutral_walk_ff_270_R_001__A534_M", + "audio_path": "/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/videogen/hfd_hesitation_pause_v2/audio/review_examples/train/repair_grab_to_left_zh_female_01_zh.wav", + "spoken_command": "向前……不对,我是说……向左走路", + "provisional_instruction": "向前走并抓取某物", + "final_instruction": "向左走路", + "ideal_motion": "整个改口过程只 listen;不得向前抓取,最终只相对当前朝向向左走。", + "repair_style": "explicit-cue" + }, + { + "turn_index": 2, + "source_motion_filename": "wave_two_hands_R_001__A533", + "audio_path": "/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/qwen3-tts/seed_audio/000155/female_01_zh.wav", + "spoken_command": "双手在身前挥动", + "ideal_motion": "第二轮正常挥手,验证最终意图替换没有破坏后续交互。" + } + ], + "ideal_control_timeline": [ + "初始 idle", + "错误前缀、改口词和完整新指令全程 listen", + "只在完整最终指令后进入 motion", + "不为错误前缀启动 motion" + ], + "continuity_requirements": [ + "旧意图与最终意图共用同一 Qwen KV,不靠清缓存作弊", + "同一音色和背景", + "动作与 RTC 只从最终意图开始" + ] +} diff --git a/scripts/infer/multiturn_auto_scenario_hesitation_repair_val_v1.json b/scripts/infer/multiturn_auto_scenario_hesitation_repair_val_v1.json new file mode 100644 index 0000000000000000000000000000000000000000..69ed0959d70b85cc4d404c879635218963b7fbae --- /dev/null +++ b/scripts/infer/multiturn_auto_scenario_hesitation_repair_val_v1.json @@ -0,0 +1,42 @@ +{ + "format": "hfd_multiturn_auto_infer_scenario_v1", + "name": "repair_pick_to_jog_then_pick_en", + "interaction_mode": "hesitation", + "interaction_subtype": "self-repair", + "data_split": "val", + "split_seed": 42, + "val_ratio": 0.1, + "split_source_namespace": "seed", + "description": "未见 motion 上先说搬运长指令的错误前缀,再用验证集 cue 改口为完整向前慢跑。", + "required_semantics_version": 23, + "initial_idle_steps": 4, + "initial_idle_seconds": 1.6, + "motion_frames_per_turn": 450, + "motion_seconds_per_turn": 9.0, + "inter_turn_background_steps_by_turn": [25], + "inter_turn_background_seconds_by_turn": [10.0], + "terminal_background_steps": 6, + "terminal_background_seconds": 2.4, + "disfluency_event": {"turn_index": 0, "subtype": "self-repair", "gap_steps": 7, "gap_seconds": 2.8, "marker_id": "repair_en_11"}, + "turns": [ + { + "turn_index": 1, + "disfluency_subtype": "self-repair", + "source_motion_filename": "jog_ff_start_180_R_003__A068", + "audio_path": "/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/videogen/hfd_hesitation_pause_v1/audio/review_examples/val/repair_pick_to_jog_en_female_02_en.wav", + "spoken_command": "picks up ... Wait, I mean ... start jog forward", + "provisional_instruction": "picks up medium big light object with two hands from low right side and puts it away to the right side high", + "final_instruction": "start jog forward", + "ideal_motion": "不得执行搬运;完整听完 start jog forward 后才向前慢跑。" + }, + { + "turn_index": 2, + "source_motion_filename": "medium_big_light_two_hands_right_side_low_to_right_side_high_R_001__A529", + "audio_path": "/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/qwen3-tts/seed_audio/000227/female_02_en.wav", + "spoken_command": "picks up medium big light object with two hands from low right side and puts it away to the right side high", + "ideal_motion": "只有第二轮真正完整重发搬运指令时,才执行右侧双手取放。" + } + ], + "ideal_control_timeline": ["初始 idle", "假开头、Wait I mean 和完整最终指令全程 listen", "第一轮只执行 jog", "第二轮完整指令后才执行 pick"], + "continuity_requirements": ["验证 cue 与训练 cue 词面分离", "Qwen KV 保留旧指令但最新完整意图应覆盖它", "两条动作与同一声音身份均保持 val 审计"] +} diff --git a/scripts/infer/multiturn_auto_scenario_hesitation_repair_val_v2.json b/scripts/infer/multiturn_auto_scenario_hesitation_repair_val_v2.json new file mode 100644 index 0000000000000000000000000000000000000000..afaea8d65988959a6427fbd81da1d81856b4b953 --- /dev/null +++ b/scripts/infer/multiturn_auto_scenario_hesitation_repair_val_v2.json @@ -0,0 +1,62 @@ +{ + "format": "hfd_multiturn_auto_infer_scenario_v1", + "name": "repair_pick_to_jog_then_pick_en", + "interaction_mode": "hesitation", + "interaction_subtype": "self-repair", + "data_split": "val", + "split_seed": 42, + "val_ratio": 0.1, + "split_source_namespace": "seed", + "description": "未见 motion 上先说搬运长指令的错误前缀,停顿后不使用 I mean 等提示词,直接重新说完整向前慢跑指令。", + "required_semantics_version": 24, + "initial_idle_steps": 4, + "initial_idle_seconds": 1.6, + "motion_frames_per_turn": 450, + "motion_seconds_per_turn": 9.0, + "inter_turn_background_steps_by_turn": [ + 25 + ], + "inter_turn_background_seconds_by_turn": [ + 10.0 + ], + "terminal_background_steps": 6, + "terminal_background_seconds": 2.4, + "disfluency_event": { + "turn_index": 0, + "subtype": "self-repair", + "gap_steps": 7, + "gap_seconds": 2.8, + "repair_style": "implicit-restart" + }, + "turns": [ + { + "turn_index": 1, + "disfluency_subtype": "self-repair", + "source_motion_filename": "jog_ff_start_180_R_003__A068", + "audio_path": "/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/videogen/hfd_hesitation_pause_v2/audio/review_examples/val/repair_pick_to_jog_en_female_02_en.wav", + "spoken_command": "picks up ... (2.8 s pause) ... start jog forward", + "provisional_instruction": "picks up medium big light object with two hands from low right side and puts it away to the right side high", + "final_instruction": "start jog forward", + "ideal_motion": "没有改口提示词也不得执行搬运;完整听完 start jog forward 后才向前慢跑。", + "repair_style": "implicit-restart" + }, + { + "turn_index": 2, + "source_motion_filename": "medium_big_light_two_hands_right_side_low_to_right_side_high_R_001__A529", + "audio_path": "/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/qwen3-tts/seed_audio/000227/female_02_en.wav", + "spoken_command": "picks up medium big light object with two hands from low right side and puts it away to the right side high", + "ideal_motion": "只有第二轮真正完整重发搬运指令时,才执行右侧双手取放。" + } + ], + "ideal_control_timeline": [ + "初始 idle", + "错误开头、停顿和重新开始的完整最终指令全程 listen", + "第一轮只执行 jog", + "第二轮完整指令后才执行 pick" + ], + "continuity_requirements": [ + "不依赖 I mean 等关键词判断改口", + "Qwen KV 保留旧指令但最新完整意图应覆盖它", + "两条动作与同一声音身份均保持 val 审计" + ] +} diff --git a/scripts/infer/multiturn_auto_scenario_interruption_train_v1.json b/scripts/infer/multiturn_auto_scenario_interruption_train_v1.json new file mode 100644 index 0000000000000000000000000000000000000000..a9d1eaf2c45a5159deb7463727a8cb2d29071a1e --- /dev/null +++ b/scripts/infer/multiturn_auto_scenario_interruption_train_v1.json @@ -0,0 +1,60 @@ +{ + "format": "hfd_multiturn_auto_infer_scenario_v1", + "name": "interrupt_grab_wave_walk_left_zh", + "interaction_mode": "interruption", + "data_split": "train", + "split_seed": 42, + "val_ratio": 0.1, + "split_source_namespace": "seed", + "description": "三轮训练集指令;第二、三条语音分别在上一动作开始约 4.0/4.8 秒后到来,用于检查因果打断、连续 KV 和连续 RTC。", + "required_semantics_version": 20, + "initial_idle_steps": 5, + "initial_idle_seconds": 2.0, + "motion_frames_per_turn": 450, + "motion_seconds_per_turn": 9.0, + "inter_turn_background_steps_by_turn": [ + 10, + 12 + ], + "inter_turn_background_seconds_by_turn": [ + 4.0, + 4.8 + ], + "terminal_background_steps": 5, + "terminal_background_seconds": 2.0, + "turns": [ + { + "turn_index": 1, + "source_motion_filename": "grab_walk_ff_180_001__A548", + "audio_path": "/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/qwen3-tts/seed_audio/000006/female_01_zh.wav", + "spoken_command": "向前走并抓取某物", + "ideal_motion": "听完后开始向前走并抓取;动作尚未结束时收到下一条指令。" + }, + { + "turn_index": 2, + "source_motion_filename": "wave_two_hands_R_001__A533", + "audio_path": "/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/qwen3-tts/seed_audio/000155/female_01_zh.wav", + "spoken_command": "双手在身前挥动", + "ideal_motion": "新语音刚出现时短暂延续旧动作,听清后平滑切入 listen,再从当时真实姿态开始挥手。" + }, + { + "turn_index": 3, + "source_motion_filename": "neutral_walk_ff_270_R_001__A534_M", + "audio_path": "/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/qwen3-tts/seed_audio/000120/female_01_zh.wav", + "spoken_command": "向左走路", + "ideal_motion": "挥手尚未自然结束时响应新语音,随后相对当前身体朝向向左行走并最终恢复站立。" + } + ], + "ideal_control_timeline": [ + "初始背景输出 ,第一条语音出现后进入 ", + "每次新语音刚开始时仍执行上一轮 ,不提前读取未来语义", + "听到有效语音后由 平滑切到 ", + "语音结束后进入新的 ,最后回到 " + ], + "continuity_requirements": [ + "三轮共用一个 Qwen KV cache 和一段连续办公室背景", + "打断不清空机器人状态或 RTC 历史", + "新动作从旧动作被打断时的真实位置、朝向和关节状态接续", + "旧动作的 future progress 仍以其原始完整时长为分母" + ] +} diff --git a/scripts/infer/multiturn_auto_scenario_pause_train_v1.json b/scripts/infer/multiturn_auto_scenario_pause_train_v1.json new file mode 100644 index 0000000000000000000000000000000000000000..8ad7a33f47a89abfcad7d9ffee467cc0444ac938 --- /dev/null +++ b/scripts/infer/multiturn_auto_scenario_pause_train_v1.json @@ -0,0 +1,41 @@ +{ + "format": "hfd_multiturn_auto_infer_scenario_v1", + "name": "pause_grab_then_wave_zh", + "interaction_mode": "pause", + "interaction_subtype": "silent-pause", + "data_split": "train", + "split_seed": 42, + "val_ratio": 0.1, + "split_source_namespace": "seed", + "description": "训练集中文指令在句中静默 1.6 秒;停顿期间继续 listen,完整意图结束后才抓取,随后正常接收第二轮指令。", + "required_semantics_version": 23, + "initial_idle_steps": 5, + "initial_idle_seconds": 2.0, + "motion_frames_per_turn": 450, + "motion_seconds_per_turn": 9.0, + "inter_turn_background_steps_by_turn": [25], + "inter_turn_background_seconds_by_turn": [10.0], + "terminal_background_steps": 5, + "terminal_background_seconds": 2.0, + "disfluency_event": {"turn_index": 0, "subtype": "silent-pause", "gap_steps": 4, "gap_seconds": 1.6}, + "turns": [ + { + "turn_index": 1, + "disfluency_subtype": "silent-pause", + "source_motion_filename": "grab_walk_ff_180_001__A548", + "audio_path": "/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/videogen/hfd_hesitation_pause_v1/audio/review_examples/train/pause_grab_zh_female_01_zh.wav", + "spoken_command": "向前……(静默思考 1.6 秒)……走并抓取某物", + "final_instruction": "向前走并抓取某物", + "ideal_motion": "静默段全程保持自然聆听,不提前走;听到完整后向前走并抓取。" + }, + { + "turn_index": 2, + "source_motion_filename": "wave_two_hands_R_001__A533", + "audio_path": "/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/qwen3-tts/seed_audio/000155/female_01_zh.wav", + "spoken_command": "双手在身前挥动", + "ideal_motion": "从第一轮结束状态自然进入第二轮双手挥动。" + } + ], + "ideal_control_timeline": ["背景为 idle", "语音开始后进入 listen", "句中静默仍保持 listen", "完整指令结束后进入 motion", "下一条完整指令按普通 turn-taking 执行"], + "continuity_requirements": ["整段 session 只使用一个 Qwen KV", "同一连续办公室背景", "RTC、位置和朝向不在停顿或轮次边界重置"] +} diff --git a/scripts/infer/multiturn_auto_scenario_pause_train_v2.json b/scripts/infer/multiturn_auto_scenario_pause_train_v2.json new file mode 100644 index 0000000000000000000000000000000000000000..d951df4c447f31c442ffd884664995e03b76823a --- /dev/null +++ b/scripts/infer/multiturn_auto_scenario_pause_train_v2.json @@ -0,0 +1,60 @@ +{ + "format": "hfd_multiturn_auto_infer_scenario_v1", + "name": "pause_grab_then_wave_zh", + "interaction_mode": "pause", + "interaction_subtype": "silent-pause", + "data_split": "train", + "split_seed": 42, + "val_ratio": 0.1, + "split_source_namespace": "seed", + "description": "训练集中文指令在句中静默 1.6 秒;停顿期间继续 listen,完整意图结束后才抓取,随后正常接收第二轮指令。", + "required_semantics_version": 24, + "initial_idle_steps": 5, + "initial_idle_seconds": 2.0, + "motion_frames_per_turn": 450, + "motion_seconds_per_turn": 9.0, + "inter_turn_background_steps_by_turn": [ + 25 + ], + "inter_turn_background_seconds_by_turn": [ + 10.0 + ], + "terminal_background_steps": 5, + "terminal_background_seconds": 2.0, + "disfluency_event": { + "turn_index": 0, + "subtype": "silent-pause", + "gap_steps": 4, + "gap_seconds": 1.6 + }, + "turns": [ + { + "turn_index": 1, + "disfluency_subtype": "silent-pause", + "source_motion_filename": "grab_walk_ff_180_001__A548", + "audio_path": "/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/videogen/hfd_hesitation_pause_v2/audio/review_examples/train/pause_grab_zh_female_01_zh.wav", + "spoken_command": "向前……(静默思考 1.6 秒)……走并抓取某物", + "final_instruction": "向前走并抓取某物", + "ideal_motion": "静默段全程保持自然聆听,不提前走;听到完整后向前走并抓取。" + }, + { + "turn_index": 2, + "source_motion_filename": "wave_two_hands_R_001__A533", + "audio_path": "/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/qwen3-tts/seed_audio/000155/female_01_zh.wav", + "spoken_command": "双手在身前挥动", + "ideal_motion": "从第一轮结束状态自然进入第二轮双手挥动。" + } + ], + "ideal_control_timeline": [ + "背景为 idle", + "语音开始后进入 listen", + "句中静默仍保持 listen", + "完整指令结束后进入 motion", + "下一条完整指令按普通 turn-taking 执行" + ], + "continuity_requirements": [ + "整段 session 只使用一个 Qwen KV", + "同一连续办公室背景", + "RTC、位置和朝向不在停顿或轮次边界重置" + ] +} diff --git a/scripts/infer/multiturn_auto_scenario_safe_hold_val_v1.json b/scripts/infer/multiturn_auto_scenario_safe_hold_val_v1.json new file mode 100644 index 0000000000000000000000000000000000000000..1c6b87626bd5e884a57836b66d40d32ee5eb2308 --- /dev/null +++ b/scripts/infer/multiturn_auto_scenario_safe_hold_val_v1.json @@ -0,0 +1,40 @@ +{ + "format": "hfd_multiturn_auto_infer_scenario_v1", + "name": "hold_val_jump_en", + "interaction_mode": "safe-hold", + "data_split": "val", + "split_seed": 42, + "val_ratio": 0.1, + "split_source_namespace": "seed", + "description": "未见过的跳跃 motion 中收到保持不动,检查是否先落地稳定再保持。", + "required_semantics_version": 21, + "initial_idle_steps": 4, + "initial_idle_seconds": 1.6, + "motion_frames_per_turn": 450, + "motion_seconds_per_turn": 9.0, + "inter_turn_background_steps_by_turn": [7], + "inter_turn_background_seconds_by_turn": [2.8], + "terminal_background_steps": 6, + "terminal_background_seconds": 2.4, + "turns": [ + { + "turn_index": 1, + "source_motion_filename": "reach_jump_R_003__A148_M", + "audio_path": "/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/qwen3-tts/seed_audio/000243/female_02_en.wav", + "spoken_command": "reaching something during jump", + "ideal_motion": "起跳并在空中伸手。" + }, + { + "turn_index": 2, + "control_intent": "safe-hold", + "audio_path": "/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/videogen/hfd_control_commands_v1/audio/safe-hold/hold_06/female_02_en.wav", + "spoken_command": "Stay balanced and don't move.", + "ideal_motion": "如果正在腾空或落地中则先完成安全落地,然后保持真实稳定姿态,不回零到预设 idle。" + } + ], + "ideal_control_timeline": [ + "idle -> listen -> motion", + "hold 语音在 motion 中到来,没有 listen", + "不稳定时保持 motion,稳定后转 idle 并保持当时姿" + ] +} diff --git a/scripts/infer/multiturn_auto_scenario_v1.json b/scripts/infer/multiturn_auto_scenario_v1.json new file mode 100644 index 0000000000000000000000000000000000000000..fbf49e651f9e8e8c39589368c1a23813f8c2ca3e --- /dev/null +++ b/scripts/infer/multiturn_auto_scenario_v1.json @@ -0,0 +1,61 @@ +{ + "format": "hfd_multiturn_auto_infer_scenario_v1", + "name": "grab_wave_walk_left_zh", + "interaction_mode": "turn-taking", + "data_split": "train", + "split_seed": 42, + "val_ratio": 0.1, + "split_source_namespace": "seed", + "description": "三轮连续中文指令;同一机器人状态、办公室背景、RTC 历史和 Qwen KV 从头保留到尾。", + "required_semantics_version": 20, + "initial_idle_steps": 5, + "initial_idle_seconds": 2.0, + "motion_frames_per_turn": 450, + "motion_seconds_per_turn": 9.0, + "inter_turn_background_steps_by_turn": [ + 25, + 30 + ], + "inter_turn_background_seconds_by_turn": [ + 10.0, + 12.0 + ], + "terminal_background_steps": 5, + "terminal_background_seconds": 2.0, + "turns": [ + { + "turn_index": 1, + "source_motion_filename": "grab_walk_ff_180_001__A548", + "audio_path": "/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/qwen3-tts/seed_audio/000006/female_01_zh.wav", + "spoken_command": "向前走并抓取某物", + "ideal_motion": "听完后向当前身体前方行走,同时完成清晰的伸手抓取;动作结束后在新位置恢复正常站立。" + }, + { + "turn_index": 2, + "source_motion_filename": "wave_two_hands_R_001__A533", + "audio_path": "/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/qwen3-tts/seed_audio/000155/female_01_zh.wav", + "spoken_command": "双手在身前挥动", + "ideal_motion": "保持第一轮累计的位置和朝向,身体基本原地站稳,双手在身前挥动,然后自然收手恢复站立。" + }, + { + "turn_index": 3, + "source_motion_filename": "neutral_walk_ff_270_R_001__A534_M", + "audio_path": "/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/qwen3-tts/seed_audio/000120/female_01_zh.wav", + "spoken_command": "向左走路", + "ideal_motion": "相对第二轮结束时的身体朝向向左侧行走,不重置到世界原点,最后恢复正常站立。" + } + ], + "ideal_control_timeline": [ + "初始纯背景时输出 ,机器人自然站立且不提前抬手或移动", + "每次有人说话时切到 ,保持自然聆听动作", + "每条指令结束后切到 ,完整执行该轮动作", + "第二轮前允许短暂停顿,第三轮前包含约 2 秒额外 idle", + "最后一轮结束后,在 2 秒纯背景中回到并持续输出 " + ], + "continuity_requirements": [ + "三轮只使用一个 Qwen KV cache,不在轮次边界清空", + "三轮使用同一段连续办公室背景,不拼接不同背景音", + "后续动作从上一轮真实生成的最终位置、朝向和关节状态继续", + "RTC 历史按真实执行的动作连续传递,不读取 GT,也不重置机器人" + ] +} diff --git a/scripts/infer/multiturn_auto_scenario_val_v1.json b/scripts/infer/multiturn_auto_scenario_val_v1.json new file mode 100644 index 0000000000000000000000000000000000000000..150bbf57b76078182dbec255d35734ee8033d7b2 --- /dev/null +++ b/scripts/infer/multiturn_auto_scenario_val_v1.json @@ -0,0 +1,100 @@ +{ + "format": "hfd_multiturn_auto_infer_scenario_v1", + "name": "val_jog_pick_jump_idle_en_timeline_v2", + "interaction_mode": "turn-taking", + "description": "训练时 hash 划分到 val 的第 1199 条四轮 session;四条具体 motion source 均未进入训练集,并严格复现该 session 的真实语音起点。", + "data_split": "val", + "split_seed": 42, + "val_ratio": 0.1, + "split_source_namespace": "seed", + "validation_session_index": 1199, + "required_semantics_version": 20, + "initial_idle_steps": 4, + "initial_idle_seconds": 1.6, + "motion_frames_per_turn": 450, + "motion_seconds_per_turn": 9.0, + "inter_turn_background_steps_by_turn": [ + 18, + 32, + 14 + ], + "inter_turn_background_seconds_by_turn": [ + 7.2, + 12.8, + 5.6 + ], + "turn_audio_window_counts": [ + 4, + 18, + 5, + 10 + ], + "expected_turn_window_starts": [ + 0, + 22, + 72, + 91 + ], + "source_turn_speech_start_frames_50fps": [ + 80, + 520, + 1520, + 1900 + ], + "source_turn_listen_start_frames_50fps": [ + 120, + 560, + 1560, + 1940 + ], + "source_inter_turn_idle_steps": [ + 0, + 8, + 0 + ], + "terminal_background_steps": 6, + "terminal_background_seconds": 2.4, + "turns": [ + { + "turn_index": 1, + "source_motion_filename": "jog_ff_start_180_R_003__A068", + "audio_path": "/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/qwen3-tts/seed_audio/000175/female_02_en.wav", + "spoken_command": "start jog forward", + "ideal_motion": "听完英文指令后从当前站立状态开始向身体前方慢跑,保持朝向连续。" + }, + { + "turn_index": 2, + "source_motion_filename": "medium_big_light_two_hands_right_side_low_to_right_side_high_R_001__A529", + "audio_path": "/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/qwen3-tts/seed_audio/000227/female_02_en.wav", + "spoken_command": "picks up medium big light object with two hands from low right side and puts it away to the right side high", + "ideal_motion": "在第一轮最终位置和朝向上,用双手从右下方拿起物体并举到右上方;这条完整指令和音频目录训练时从未出现。" + }, + { + "turn_index": 3, + "source_motion_filename": "reach_jump_R_003__A148_M", + "audio_path": "/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/qwen3-tts/seed_audio/000243/female_02_en.wav", + "spoken_command": "reaching something during jump", + "ideal_motion": "保持累计状态,完成带伸手够物意图的跳跃并稳定落地。" + }, + { + "turn_index": 4, + "source_motion_filename": "idle_right_to_idle_003__A094_M", + "audio_path": "/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/qwen3-tts/seed_audio/003713/female_02_en.wav", + "spoken_command": "from neutral position with left leg forward to neutral position", + "ideal_motion": "从左腿略在前的自然站姿平稳回到中立站姿,随后在纯背景中回到 idle。" + } + ], + "ideal_control_timeline": [ + "初始纯背景输出 ", + "每轮语音出现后切换到 ", + "听清完整指令后切换到 ", + "四轮语音起点严格复现验证 source 的 0、8.8、28.8、36.4 秒相对时间;7.2、12.8、5.6 秒是扣除上一条语音窗口后的实际背景间隔", + "第四轮结束后在 2.4 秒背景中回到 " + ], + "continuity_requirements": [ + "四轮共用一个 Qwen KV cache", + "四轮使用同一段连续办公室背景", + "机器人位置、朝向、关节状态和 RTC 历史不在轮次间重置", + "所有 source_motion_filename 均按训练 run 的 split_seed=42、val_ratio=0.1 落在 val" + ] +} diff --git a/scripts/infer/multiturn_auto_suite_v2.json b/scripts/infer/multiturn_auto_suite_v2.json new file mode 100644 index 0000000000000000000000000000000000000000..6948abb4276531dbd6f7af70ce8ad84b34480af4 --- /dev/null +++ b/scripts/infer/multiturn_auto_suite_v2.json @@ -0,0 +1,92 @@ +{ + "format": "hfd_multiturn_auto_infer_suite_v2", + "description": "同一 checkpoint 的既有五类任务及 pause、filled hesitation、self-repair hesitation × train/val 共十六组闭环推理;由空闲 GPU 动态领取任务。", + "cases": [ + { + "interaction_mode": "turn-taking", + "data_split": "train", + "scenario_file": "multiturn_auto_scenario_v1.json" + }, + { + "interaction_mode": "turn-taking", + "data_split": "val", + "scenario_file": "multiturn_auto_scenario_val_v1.json" + }, + { + "interaction_mode": "interruption", + "data_split": "train", + "scenario_file": "multiturn_auto_scenario_interruption_train_v1.json" + }, + { + "interaction_mode": "interruption", + "data_split": "val", + "scenario_file": "multiturn_auto_scenario_interruption_val_v1.json" + }, + { + "interaction_mode": "emergency-stop", + "data_split": "train", + "scenario_file": "multiturn_auto_scenario_emergency_stop_train_v1.json" + }, + { + "interaction_mode": "emergency-stop", + "data_split": "val", + "scenario_file": "multiturn_auto_scenario_emergency_stop_val_v1.json" + }, + { + "interaction_mode": "safe-hold", + "data_split": "train", + "scenario_file": "multiturn_auto_scenario_safe_hold_train_v1.json" + }, + { + "interaction_mode": "safe-hold", + "data_split": "val", + "scenario_file": "multiturn_auto_scenario_safe_hold_val_v1.json" + }, + { + "interaction_mode": "backchannel", + "data_split": "train", + "scenario_file": "multiturn_auto_scenario_backchannel_train_v1.json" + }, + { + "interaction_mode": "backchannel", + "data_split": "val", + "scenario_file": "multiturn_auto_scenario_backchannel_val_v1.json" + }, + { + "interaction_mode": "pause", + "interaction_subtype": "silent-pause", + "data_split": "train", + "scenario_file": "multiturn_auto_scenario_pause_train_v1.json" + }, + { + "interaction_mode": "pause", + "interaction_subtype": "silent-pause", + "data_split": "val", + "scenario_file": "multiturn_auto_scenario_pause_val_v1.json" + }, + { + "interaction_mode": "hesitation", + "interaction_subtype": "filled-pause", + "data_split": "train", + "scenario_file": "multiturn_auto_scenario_hesitation_filled_train_v1.json" + }, + { + "interaction_mode": "hesitation", + "interaction_subtype": "filled-pause", + "data_split": "val", + "scenario_file": "multiturn_auto_scenario_hesitation_filled_val_v1.json" + }, + { + "interaction_mode": "hesitation", + "interaction_subtype": "self-repair", + "data_split": "train", + "scenario_file": "multiturn_auto_scenario_hesitation_repair_train_v1.json" + }, + { + "interaction_mode": "hesitation", + "interaction_subtype": "self-repair", + "data_split": "val", + "scenario_file": "multiturn_auto_scenario_hesitation_repair_val_v1.json" + } + ] +} diff --git a/scripts/infer/multiturn_auto_suite_v3.json b/scripts/infer/multiturn_auto_suite_v3.json new file mode 100644 index 0000000000000000000000000000000000000000..321dad0408c36966780b95769414d1dae462f984 --- /dev/null +++ b/scripts/infer/multiturn_auto_suite_v3.json @@ -0,0 +1,92 @@ +{ + "format": "hfd_multiturn_auto_infer_suite_v3", + "description": "同一 checkpoint 的七类任务 × train/val 共十六组闭环推理;v24 使用自然拖长 filler,并同时覆盖显式提示和无提示词自我改口;由空闲 GPU 动态领取任务。", + "cases": [ + { + "interaction_mode": "turn-taking", + "data_split": "train", + "scenario_file": "multiturn_auto_scenario_v1.json" + }, + { + "interaction_mode": "turn-taking", + "data_split": "val", + "scenario_file": "multiturn_auto_scenario_val_v1.json" + }, + { + "interaction_mode": "interruption", + "data_split": "train", + "scenario_file": "multiturn_auto_scenario_interruption_train_v1.json" + }, + { + "interaction_mode": "interruption", + "data_split": "val", + "scenario_file": "multiturn_auto_scenario_interruption_val_v1.json" + }, + { + "interaction_mode": "emergency-stop", + "data_split": "train", + "scenario_file": "multiturn_auto_scenario_emergency_stop_train_v1.json" + }, + { + "interaction_mode": "emergency-stop", + "data_split": "val", + "scenario_file": "multiturn_auto_scenario_emergency_stop_val_v1.json" + }, + { + "interaction_mode": "safe-hold", + "data_split": "train", + "scenario_file": "multiturn_auto_scenario_safe_hold_train_v1.json" + }, + { + "interaction_mode": "safe-hold", + "data_split": "val", + "scenario_file": "multiturn_auto_scenario_safe_hold_val_v1.json" + }, + { + "interaction_mode": "backchannel", + "data_split": "train", + "scenario_file": "multiturn_auto_scenario_backchannel_train_v1.json" + }, + { + "interaction_mode": "backchannel", + "data_split": "val", + "scenario_file": "multiturn_auto_scenario_backchannel_val_v1.json" + }, + { + "interaction_mode": "pause", + "interaction_subtype": "silent-pause", + "data_split": "train", + "scenario_file": "multiturn_auto_scenario_pause_train_v2.json" + }, + { + "interaction_mode": "pause", + "interaction_subtype": "silent-pause", + "data_split": "val", + "scenario_file": "multiturn_auto_scenario_pause_val_v2.json" + }, + { + "interaction_mode": "hesitation", + "interaction_subtype": "filled-pause", + "data_split": "train", + "scenario_file": "multiturn_auto_scenario_hesitation_filled_train_v2.json" + }, + { + "interaction_mode": "hesitation", + "interaction_subtype": "filled-pause", + "data_split": "val", + "scenario_file": "multiturn_auto_scenario_hesitation_filled_val_v2.json" + }, + { + "interaction_mode": "hesitation", + "interaction_subtype": "self-repair", + "data_split": "train", + "scenario_file": "multiturn_auto_scenario_hesitation_repair_train_v2.json" + }, + { + "interaction_mode": "hesitation", + "interaction_subtype": "self-repair", + "data_split": "val", + "scenario_file": "multiturn_auto_scenario_hesitation_repair_val_v2.json" + } + ] +} diff --git a/scripts/infer/mux_full_duplex_audio.py b/scripts/infer/mux_full_duplex_audio.py new file mode 100644 index 0000000000000000000000000000000000000000..21294cdee8f05b806996c571dc5752159dc751cc --- /dev/null +++ b/scripts/infer/mux_full_duplex_audio.py @@ -0,0 +1,180 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import argparse +import json +import subprocess +from pathlib import Path +from typing import Any + + +def probe(path: Path) -> dict[str, Any]: + result = subprocess.run( + [ + "ffprobe", + "-v", + "error", + "-show_streams", + "-show_format", + "-of", + "json", + str(path), + ], + capture_output=True, + check=False, + ) + if result.returncode != 0: + raise RuntimeError(result.stderr.decode("utf-8", errors="ignore")) + return json.loads(result.stdout) + + +def stream(payload: dict[str, Any], codec_type: str) -> dict[str, Any]: + for item in payload.get("streams", []): + if item.get("codec_type") == codec_type: + return item + raise RuntimeError(f"ffprobe found no {codec_type} stream") + + +def fraction(value: str) -> float: + numerator, denominator = str(value).split("/", 1) + return float(numerator) / float(denominator) + + +def main() -> None: + parser = argparse.ArgumentParser( + description=( + "Mux either a complete HFD streaming audio timeline or legacy " + "speech-only audio onto the rendered motion" + ) + ) + parser.add_argument("--video", required=True) + parser.add_argument("--audio", required=True) + parser.add_argument("--output", required=True) + parser.add_argument("--infer-meta", required=True) + parser.add_argument( + "--audio-is-timeline", + action="store_true", + help=( + "The audio already contains initial background, speech, and " + "post-WAV background on the complete 50 Hz output timeline." + ), + ) + parser.add_argument("--expected-fps", type=float, default=50.0) + args = parser.parse_args() + + video = Path(args.video).expanduser().resolve() + audio = Path(args.audio).expanduser().resolve() + output = Path(args.output).expanduser().resolve() + meta_path = Path(args.infer_meta).expanduser().resolve() + for path in (video, audio, meta_path): + if not path.is_file(): + raise FileNotFoundError(path) + + video_probe = probe(video) + video_stream = stream(video_probe, "video") + video_duration = float( + video_stream.get("duration") or video_probe["format"].get("duration") + ) + video_fps = fraction(video_stream.get("avg_frame_rate") or video_stream["r_frame_rate"]) + if abs(video_fps - float(args.expected_fps)) > 0.01: + raise RuntimeError( + f"Rendered video FPS {video_fps} does not preserve the 50 Hz HFD timeline" + ) + + metadata = json.loads(meta_path.read_text(encoding="utf-8")) + requested_speech_delay_seconds = max( + float(metadata.get("speech_start_seconds", 0.0)), + 0.0, + ) + applied_audio_delay_seconds = ( + 0.0 if bool(args.audio_is_timeline) else requested_speech_delay_seconds + ) + speech_delay_ms = int(round(applied_audio_delay_seconds * 1000.0)) + delay_filter = ( + f"adelay={speech_delay_ms}:all=1," + if speech_delay_ms > 0 + else "" + ) + output.parent.mkdir(parents=True, exist_ok=True) + temporary = output.with_name(f".{output.name}.tmp.mp4") + command = [ + "ffmpeg", + "-v", + "error", + "-y", + "-i", + str(video), + "-i", + str(audio), + "-filter_complex", + f"[1:a]{delay_filter}aresample=async=1:first_pts=0," + f"apad=whole_dur={video_duration:.9f}[audio]", + "-map", + "0:v:0", + "-map", + "[audio]", + "-c:v", + "copy", + "-c:a", + "aac", + "-movflags", + "+faststart", + "-t", + f"{video_duration:.9f}", + str(temporary), + ] + result = subprocess.run(command, capture_output=True, check=False) + if result.returncode != 0: + raise RuntimeError( + f"ffmpeg mux failed: {result.stderr.decode('utf-8', errors='ignore')}" + ) + temporary.replace(output) + + output_probe = probe(output) + output_video = stream(output_probe, "video") + output_audio = stream(output_probe, "audio") + video_start = float(output_video.get("start_time", 0.0)) + audio_start = float(output_audio.get("start_time", 0.0)) + output_duration = float(output_probe["format"].get("duration", 0.0)) + frame_tolerance = 1.0 / float(args.expected_fps) + 1e-3 + if abs(video_start) > frame_tolerance or abs(audio_start) > frame_tolerance: + raise RuntimeError( + f"Muxed streams do not start together: video={video_start}, audio={audio_start}" + ) + if output_duration + frame_tolerance < video_duration: + raise RuntimeError( + f"Mux truncated the motion video: input={video_duration}, output={output_duration}" + ) + + metadata["mux"] = { + "video_without_audio": str(video), + "video_with_audio": str(output), + "audio": str(audio), + "video_fps": video_fps, + "video_start_seconds": video_start, + "audio_start_seconds": audio_start, + "video_duration_seconds": video_duration, + "muxed_duration_seconds": output_duration, + "audio_mode": ( + "complete_stream_timeline" + if bool(args.audio_is_timeline) + else "speech_only_legacy" + ), + "audio_padding": ( + "timeline_apad_only_if_container_rounding_requires_it" + if bool(args.audio_is_timeline) + else "silence_to_video_end" + ), + "speech_delay_seconds": requested_speech_delay_seconds, + "applied_audio_delay_seconds": applied_audio_delay_seconds, + "motion_start_frame_50fps": metadata.get("motion_start_frame_50fps"), + "motion_start_seconds": metadata.get("motion_start_seconds"), + } + meta_path.write_text( + json.dumps(metadata, ensure_ascii=False, indent=2) + "\n", encoding="utf-8" + ) + print(f"[done] synchronized video: {output}", flush=True) + + +if __name__ == "__main__": + main() diff --git a/scripts/infer/render_third_party_review.py b/scripts/infer/render_third_party_review.py new file mode 100644 index 0000000000000000000000000000000000000000..27033a8ea220c0a5844edf3fbb27a6446aac4771 --- /dev/null +++ b/scripts/infer/render_third_party_review.py @@ -0,0 +1,287 @@ +#!/usr/bin/env python3 +"""Render a stratified final-model review from saved predictions, CPU only.""" +from __future__ import annotations + +import argparse +from concurrent.futures import ThreadPoolExecutor, as_completed +import hashlib +import json +import os +from pathlib import Path +import random +import shutil +import subprocess +import sys +import time +import wave + +PROJECT = Path(__file__).resolve().parents[2] +sys.path[:0] = [str(PROJECT), str(PROJECT / "src")] +PROFILES = ("single_short", "multi_single", "multi_babble", "mixed_short", "continuous", "same_voice_aside") +LABELS = dict(zip(PROFILES, ("单段短人声", "多段单人声", "多段多人杂谈", "单/多人混合短声", "连续背景说话", "指令同音色旁话"))) +ASSETS = ("motion.npy", "body_pos.csv", "body_quat.csv", "joint_pos.csv", "body_lin_vel.csv", + "body_ang_vel.csv", "joint_vel.csv", "metadata.txt", "infer_meta.json", "metrics.json", "session_recipe.json") + + +def read(path): + return json.loads(Path(path).read_text()) + + +def digest(path): + h = hashlib.sha256() + with Path(path).open("rb") as handle: + for block in iter(lambda: handle.read(1024 * 1024), b""): + h.update(block) + return h.hexdigest() + + +def write(path, value): + path = Path(path) + temp = path.with_suffix(path.suffix + ".tmp") + temp.write_text(json.dumps(value, ensure_ascii=False, indent=2, allow_nan=False) + "\n") + temp.replace(path) + + +def select_samples(selection, report, seed=26100801): + rows, samples = selection["sessions"], report["samples"] + ids = [r["session_id"] for r in rows] + sample_ids = [r["session_id"] for r in samples] + if (report.get("status") != "complete" or report.get("evaluation_split") != "train_only" + or report.get("count") != 300 or len(rows) != 300 or len(samples) != 300 + or len(set(ids)) != 300 or len(set(sample_ids)) != 300 + or set(ids) != set(sample_ids) or selection["session_ids"] != ids): + raise ValueError("Expected the same complete unique train300 selection and report") + by_id = {r["session_id"]: r for r in rows} + for sample in samples: + row = by_id[sample["session_id"]] + if (sample["profile"] != row["session_type"] + or sample["background_language"] != row["background_language"]): + raise ValueError("Prediction profile/language differs from frozen selection") + rng, chosen = random.Random(seed), [] + for profile in PROFILES: + for language in ("zh", "en"): + for high in (False, True): + pool = sorted((s for s in samples if s["profile"] == profile + and s["background_language"] == language + and by_id[s["session_id"]]["background_high"] == high), key=lambda s: s["session_id"]) + if not pool: + raise ValueError(f"Empty review cell: {profile}/{language}/{high}") + chosen.append({**rng.choice(pool), "number": f"{len(chosen) + 1:02d}", "background_high": high}) + return chosen + + +def contained(path, root): + path, root = Path(path).resolve(strict=True), Path(root).resolve(strict=True) + if path == root or not path.is_relative_to(root): + raise ValueError(f"Path escaped expected input root: {path}") + return path + + +def render_session_alive(pid): + """Include orphaned encoders/renderers, not only the coordinating worker.""" + for stat in Path("/proc").glob("[0-9]*/stat"): + try: + fields = stat.read_text().rsplit(")", 1)[1].split() + except (FileNotFoundError, ProcessLookupError): + continue + if fields[0] != "Z" and int(fields[3]) == int(pid): + return True + return False + + +def prepare(run, manifest): + import numpy as np + run = contained(run, PROJECT / "runs") + report_path = run / "final_train300/report.json" + report, result, selection = read(report_path), read(run / "result.json"), read(manifest) + if result["status"] != "complete" or report["step"] != result["step"]: + raise ValueError("Final report does not describe the completed run") + status = read(run / "run_status.json") + if digest(manifest) != status["manifest_sha256"]: + raise ValueError("Frozen train300 selection hash changed") + chosen = select_samples(selection, report) + for row in chosen: + source = contained(row["prediction_directory"], run / "final_train300/samples") + meta = read(source / "infer_meta.json") + recipe = read(source / "session_recipe.json") + if (meta["session_id"] != row["session_id"] or meta["global_step"] != report["step"] + or meta["artifact_role"] != "model_prediction_not_GT" + or recipe["split"] != "train" or recipe["session_id"] != row["session_id"]): + raise ValueError("Prediction provenance is inconsistent") + audio = contained(meta["frozen_audio_path"], PROJECT / "data/hfd_third_party_multi_v5/production/audio/train") + if digest(audio) != meta["frozen_audio_sha256"] or meta["frozen_audio_sha256"] != recipe["audio"]["sha256"]: + raise ValueError("Frozen mixed audio changed") + with wave.open(str(audio), "rb") as wav: + if (wav.getframerate(), wav.getnchannels(), wav.getsampwidth()) != (16000, 1, 2): + raise ValueError("Expected the original mono 16 kHz PCM16") + frames, seconds = round(wav.getnframes() * 50 / 16000), wav.getnframes() / 16000 + motion = np.load(source / "motion.npy", allow_pickle=False) + if motion.shape != (frames, 36) or not np.isfinite(motion).all() or digest(source / "motion.npy") != row["motion_sha256"]: + raise ValueError("Saved predicted trajectory is incomplete or changed") + for name, width in (("body_pos.csv", 3), ("body_quat.csv", 4), ("joint_pos.csv", 29)): + values = np.loadtxt(source / name, delimiter=",", skiprows=1, ndmin=2) + if values.shape != (frames, width) or not np.isfinite(values).all(): + raise ValueError(f"Render CSV has invalid length or values: {name}") + row.update(source_directory=str(source), audio_path=str(audio), audio_sha256=meta["frozen_audio_sha256"], + duration_seconds=seconds, source_files={name: digest(source / name) for name in ASSETS}) + return {"schema": 1, "run_dir": str(run), "step": report["step"], "artifact_role": "model_prediction_not_GT", + "evaluation_split": "train_only", "seed": 26100801, "count": 24, "inference_repeated": False, + "report_sha256": digest(report_path), "manifest_sha256": digest(manifest), + "selection_policy": "one random sample per profile/language/loudness cell; never select by score", + "render": {"backend": "mesa_cpu", "workers": 4, "threads_each": 4, "fps": 25, "width": 320, "height": 240}, + "audio_policy": "original mixed timeline; no gain/offset change; AAC encoding in MP4", + "samples": chosen} + + +def render_one(row, output): + from scripts.infer.third_party_checkpoint_probe import render_prediction + from scripts.infer.mux_full_duplex_audio import probe, stream, fraction + started = time.monotonic() + source, audio = Path(row["source_directory"]), Path(row["audio_path"]) + destination = output / "samples" / row["number"] + flat = output / f"{row['number']}.mp4" + saved_result = destination / "render_result.json" + for path in (output, output / "samples", destination, flat, saved_result, *(destination / name for name in ASSETS)): + if path.is_symlink(): + raise ValueError("Refusing symlink in dedicated preview outputs") + if saved_result.exists(): + result = read(saved_result) + if (result.get("status") != "complete" or any(result.get(key) != value for key, value in row.items()) + or not result.get("full_decode_passed") or not result.get("source_files_unchanged") + or result.get("video") != str(flat) or digest(flat) != result["video_sha256"] + or flat.stat().st_size != result.get("video_bytes")): + raise ValueError("Completed preview does not match the frozen render plan") + if any(digest(source / name) != expected for name, expected in row["source_files"].items()) or digest(audio) != row["audio_sha256"]: + raise ValueError("Original prediction/audio changed before preview reuse") + return {**result, "reused_existing_video": True} + if flat.exists(): + raise ValueError("Unverified video exists without a completed render result; refusing overwrite") + # Only unfinished copies in this dedicated output may be rewritten on resume. + destination.mkdir(parents=True, exist_ok=True) + for name, expected in row["source_files"].items(): + if digest(source / name) != expected: + raise ValueError("Source prediction changed after preflight") + shutil.copy2(source / name, destination / name) + if digest(destination / name) != expected: + raise ValueError("Prediction copy changed") + if digest(audio) != row["audio_sha256"]: + raise ValueError("Audio changed after preflight") + video = render_prediction(destination, audio) + media = probe(video); v, a = stream(media, "video"), stream(media, "audio") + if (v["width"], v["height"]) != (320, 240) or abs(fraction(v["avg_frame_rate"]) - 25) > .01: + raise ValueError("Rendered dimensions/FPS changed") + if int(a["channels"]) != 1 or int(a["sample_rate"]) != 16000: + raise ValueError("Rendered audio stream format changed") + if float(media["format"]["duration"]) < row["duration_seconds"] - .041: + raise ValueError("Rendered video truncated the input timeline") + for name, expected in row["source_files"].items(): + if digest(source / name) != expected: + raise ValueError("Original prediction was modified during rendering") + if digest(audio) != row["audio_sha256"]: + raise ValueError("Original audio was modified during rendering") + # A second name for the same local output; no duplicate video storage. + os.link(video, flat) + result = {**row, "status": "complete", "video": str(flat), "video_sha256": digest(flat), + "video_bytes": flat.stat().st_size, "full_decode_passed": True, "source_files_unchanged": True, + "render_seconds": time.monotonic() - started} + write(destination / "render_result.json", result) + return result + + +def publish(output, plan, completed): + lines = ["# 最终模型预测视频(不是 GT)", "", f"Checkpoint **{plan['step']}**;已完成 **{len(completed)}/24**。", + "", "复用已保存的训练集预测,不重复推理;六类×中英文背景×普通/较高背景音量,每格固定随机抽一条。", + "320×240、25fps审核预览;原始预测仍为50Hz。完整原混音不调音量、不改时间位置,MP4音轨编码为AAC。", + "", "下表指标对应各自单条样本,仅供定位问题,不作为总体效果估计。", "", + "| 序号/视频 | 类型 | 背景语言 | 背景音量 | 时长 | 控制准确率 | 背景误动率 |", + "|---|---|---|---|---:|---:|---:|"] + by_number = {r["number"]: r for r in completed} + for row in plan["samples"]: + number = row["number"]; label = f"[{number}]({number}.mp4)" if number in by_number else f"{number} 待完成" + metrics = row["metrics"]; bg = metrics.get("background_only_false_motion_rate") + bg = "无可评分区间" if bg is None else f"{bg:.1%}" + lines.append(f"| {label} | {LABELS[row['profile']]} | {'中文' if row['background_language']=='zh' else '英文'} | " + f"{'较高' if row['background_high'] else '普通'} | {row['duration_seconds']:.1f}s | {metrics['control_accuracy']:.1%} | {bg} |") + lines.extend(["", "[原始逐条评估](../final_train300/report.json) · [抽样与来源校验](plan.json) · [渲染状态](status.json)", ""]) + (output / "README.md").write_text("\n".join(lines)) + run = Path(plan["run_dir"]); review = run / "review.md" + if review.exists(): + marker = "" + text = review.read_text().split(marker)[0].rstrip() + text = text.replace("| — | 尚无完整 probe 记录 | — | — | — | — |", + "| — | 本轮采用轮末数值评估,见下方逐轮表 | — | — | — | [预测视频](prediction_videos/README.md) |") + review.write_text(text + f"\n\n{marker}\n## 最终模型预测视频\n\n" + f"Checkpoint {plan['step']},已渲染 {len(completed)}/24 条,**模型预测,不是GT**。\n\n" + "[01–24 视频与类型表](prediction_videos/README.md) · [最终300条评估](final_train300/report.json)\n") + + +def worker(output): + # Imports occur before parallel rendering; no CUDA model is loaded. + from scripts.infer.third_party_checkpoint_probe import check_renderer + check_renderer() + plan = read(output / "plan.json") + completed, failures = [], [] + started = time.time() + def status(phase): + write(output / "status.json", {"phase": phase, "pid": os.getpid(), "started_at": started, + "updated_at": time.time(), "completed": len(completed), "total": 24, "failed": len(failures), + "samples": completed, "errors": failures, "gpu_used": False, "inference_repeated": False, + "workers": plan["render"]["workers"]}) + publish(output, plan, completed) + status("rendering") + with ThreadPoolExecutor(max_workers=plan["render"]["workers"]) as pool: + jobs = {pool.submit(render_one, row, output): row for row in plan["samples"]} + for future in as_completed(jobs): + row = jobs[future] + try: + result = future.result(); completed.append(result) + print(json.dumps({"number": row["number"], "seconds": result["render_seconds"], "status": "complete"}), flush=True) + except Exception as error: + failures.append({"number": row["number"], "error": repr(error)}) + print(json.dumps(failures[-1]), flush=True) + status("rendering") + status("failed" if failures else "completed") + if failures: + raise RuntimeError(f"{len(failures)} preview renders failed; inspect per-sample render.log") + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--run-dir", type=Path, required=True) + parser.add_argument("--manifest", type=Path, default=PROJECT / "data/hfd_third_party_multi_v5/training_pilot_300/selection.json") + parser.add_argument("--workers", type=int, choices=range(1, 17), default=12) + parser.add_argument("--resume", action="store_true", help="Reuse verified videos after the previous render worker has exited") + mode = parser.add_mutually_exclusive_group(required=True) + for name in ("launch", "worker", "check-only"): + mode.add_argument(f"--{name}", action="store_true") + args = parser.parse_args(); run = contained(args.run_dir, PROJECT / "runs"); output = run / "prediction_videos" + if args.resume and not args.launch: + parser.error("--resume requires --launch") + os.environ.update(CUDA_VISIBLE_DEVICES="", PYTHONDONTWRITEBYTECODE="1", OMP_NUM_THREADS="1", MKL_NUM_THREADS="1") + if args.worker: + worker(output); return + plan = prepare(run, args.manifest) + plan["render"]["workers"] = args.workers + if args.check_only: + print(json.dumps({"passed": True, "step": plan["step"], "count": 24, "seconds": sum(r['duration_seconds'] for r in plan['samples'])})); return + if args.resume: + previous = read(output / "plan.json") + if any(previous[key] != plan[key] for key in plan if key != "render"): + raise ValueError("Cannot resume a different frozen prediction review") + if any(previous["render"][key] != plan["render"][key] for key in plan["render"] if key != "workers"): + raise ValueError("Resume can change worker count only, not rendering settings") + previous_status = read(output / "status.json") + if render_session_alive(previous_status["pid"]): + raise ValueError("Previous render session is still present; refusing concurrent launch") + else: + output.mkdir(exist_ok=False) + write(output / "plan.json", plan) + with (output / "render.log").open("a" if args.resume else "x") as log: + child = subprocess.Popen([sys.executable, str(Path(__file__).resolve()), "--run-dir", str(run), "--worker"], + cwd=PROJECT, env=os.environ.copy(), stdin=subprocess.DEVNULL, stdout=log, stderr=subprocess.STDOUT, + start_new_session=True) + print(json.dumps({"render_pid": child.pid, "output": str(output), "index": str(output / 'README.md')})) + + +if __name__ == "__main__": + main() diff --git a/scripts/infer/render_third_party_scale.py b/scripts/infer/render_third_party_scale.py new file mode 100644 index 0000000000000000000000000000000000000000..c229c4c44339168a1416ea4611aae09a6ad39874 --- /dev/null +++ b/scripts/infer/render_third_party_scale.py @@ -0,0 +1,151 @@ +#!/usr/bin/env python3 +"""CPU-only final scale previews, with separate train and held-out indexes.""" +from __future__ import annotations + +import argparse +from concurrent.futures import ThreadPoolExecutor, as_completed +import os +from pathlib import Path +import sys +import time +import wave + +PROJECT = Path(__file__).resolve().parents[2] +sys.path[:0] = [str(PROJECT), str(PROJECT / "src")] +from scripts.infer.render_third_party_review import ( + ASSETS, LABELS, contained, digest, read, render_one, write) + + +def selected_rows(selection, report, step): + if (selection.get("scaleup") is not True or selection.get("split") not in ("train", "validation") + or report.get("status") != "complete" or report.get("step") != step + or report.get("evaluation_split") != selection["split"]): + raise ValueError("Final prediction split/step differs from its frozen selection") + by_id = {r["session_id"]: r for r in selection["sessions"]} + samples = {r["session_id"]: r for r in report["samples"]} + if len(samples) != report["count"] or len(samples) != len(report["samples"]): + raise ValueError("Duplicate/incomplete final predictions") + ids = selection["video_session_ids"] + if not ids or len(ids) != len(set(ids)) or not set(ids) <= samples.keys() & by_id.keys(): + raise ValueError("Missing predetermined video samples") + rows = [] + for index, sid in enumerate(ids, 1): + row, chosen = samples[sid], by_id[sid] + if (row["profile"] != chosen["session_type"] or row["split"] != selection["split"] + or row["background_language"] != chosen["background_language"]): + raise ValueError("Prediction labels differ from frozen selection") + rows.append({**row, "number": f"{index:02d}", "background_high": chosen["background_high"]}) + return rows + + +def prepare(run, manifest): + import numpy as np + run = contained(run, PROJECT / "runs") + result, identity = read(run / "result.json"), read(run / "scale_identity.json") + if result["status"] != "complete": + raise ValueError("Training/evaluation has not completed") + rows, hashes = [], {} + for split in ("train", "validation"): + path = manifest if split == "train" else manifest.parent / "validation.json" + key = "manifest_sha256" if split == "train" else "validation_sha256" + if digest(path) != identity[key]: + raise ValueError("Frozen video selection changed") + selection = read(path) + report_path = run / "evaluations/final" / split / "report.json" + report = read(report_path); hashes[split] = digest(report_path) + for row in selected_rows(selection, report, result["step"]): + source = contained(row["prediction_directory"], report_path.parent / "samples") + meta, recipe = read(source / "infer_meta.json"), read(source / "session_recipe.json") + if (meta["session_id"] != row["session_id"] or meta["global_step"] != result["step"] + or meta.get("evaluation_split") != split or meta["artifact_role"] != "model_prediction_not_GT" + or recipe["session_id"] != row["session_id"] or recipe["split"] != split): + raise ValueError("Prediction provenance mismatch") + audio = contained(meta["frozen_audio_path"], Path(selection["production_root"]) / "audio" / split) + if digest(audio) != meta["frozen_audio_sha256"] or digest(audio) != recipe["audio"]["sha256"]: + raise ValueError("Original mixed audio changed") + with wave.open(str(audio), "rb") as wav: + if (wav.getframerate(), wav.getnchannels(), wav.getsampwidth()) != (16000, 1, 2): + raise ValueError("Expected original mono 16kHz PCM16") + seconds = wav.getnframes() / 16000 + frames = round(seconds * 50) + motion = np.load(source / "motion.npy", allow_pickle=False) + if motion.shape != (frames, 36) or not np.isfinite(motion).all() or digest(source / "motion.npy") != row["motion_sha256"]: + raise ValueError("Prediction trajectory changed or incomplete") + for name, width in (("body_pos.csv", 3), ("body_quat.csv", 4), ("joint_pos.csv", 29)): + values = np.loadtxt(source / name, delimiter=",", skiprows=1, ndmin=2) + if values.shape != (frames, width) or not np.isfinite(values).all(): + raise ValueError(f"Invalid predicted render CSV: {name}") + rows.append({**row, "source_directory": str(source), "audio_path": str(audio), + "audio_sha256": meta["frozen_audio_sha256"], "duration_seconds": seconds, + "source_files": {name: digest(source / name) for name in ASSETS}}) + return {"schema": 1, "run_dir": str(run), "step": result["step"], "count": len(rows), "samples": rows, + "report_hashes": hashes, "gpu_used": False, "inference_repeated": False, + "selection_policy": "predetermined one per available class/language/loudness cell, never by model score", + "artifact_role": "model_prediction_not_GT", "render": {"workers": 12, "fps": 25, "size": [320, 240]}} + + +def publish(output, plan, completed): + done = {(r["split"], r["number"]) for r in completed} + lines = ["# 全量训练最终预测视频(不是 GT)", "", + f"新阶段 checkpoint {plan['step']};完成 {len(completed)}/{plan['count']}。", + "复用完整预测;原混音不改变音量和时间。CPU 并行渲染,320×240、25fps;原预测仍为50Hz。", + "样例在训练前按六类×中英文背景×普通/较高音量固定随机选定,不按效果挑选。", ""] + for split, label in (("train", "训练集(拟合)"), ("validation", "独立验证集(未参与优化)")): + lines += [f"## {label}", "", "| 视频序号 | 类型 | 背景语言 | 背景音量 | 控制准确率 | 背景误动率 |", + "|---|---|---|---|---:|---:|"] + for row in plan["samples"]: + if row["split"] != split: + continue + number = row["number"] + link = f"[{number}]({split}/{number}.mp4)" if (split, number) in done else f"{number} 待生成" + metrics = row["metrics"]; bg = metrics.get("background_only_false_motion_rate") + bg_text = "无评分区间" if bg is None else f"{bg:.1%}" + lines.append(f"| {link} | {LABELS[row['profile']]} | {'中文' if row['background_language']=='zh' else '英文'} | " + f"{'较高' if row['background_high'] else '普通'} | {metrics['control_accuracy']:.1%} | {bg_text} |") + lines += ["", f"[本组原始逐条指标](../evaluations/final/{split}/report.json)", ""] + lines += ["单条视频指标用于定位问题,不代替全部验证结果,也不代表机器人实际执行成功。", "", + "[渲染状态](status.json) · [完整抽样溯源](plan.json)", ""] + (output / "README.md").write_text("\n".join(lines)) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--run-dir", type=Path, required=True) + parser.add_argument("--manifest", type=Path, required=True) + args = parser.parse_args() + os.environ.update(CUDA_VISIBLE_DEVICES="", OMP_NUM_THREADS="1", MKL_NUM_THREADS="1", PYTHONDONTWRITEBYTECODE="1") + from scripts.infer.third_party_checkpoint_probe import check_renderer + plan = prepare(args.run_dir, args.manifest.resolve()) + output = Path(plan["run_dir"]) / "prediction_videos" + output.mkdir(exist_ok=False) + write(output / "plan.json", plan) + for split in ("train", "validation"): + (output / split).mkdir() + completed, failures = [], [] + def status(phase): + write(output / "status.json", {"phase": phase, "time": time.time(), "pid": os.getpid(), + "completed": len(completed), "total": plan["count"], "failures": failures, + "gpu_used": False, "inference_repeated": False}) + publish(output, plan, completed) + status("starting") + try: + check_renderer() + with ThreadPoolExecutor(max_workers=12) as pool: + jobs = {pool.submit(render_one, row, output / row["split"]): row for row in plan["samples"]} + for future in as_completed(jobs): + row = jobs[future] + try: + completed.append(future.result()) + print(f"{row['split']}/{row['number']} completed", flush=True) + except Exception as error: + failures.append({"split": row["split"], "number": row["number"], "error": repr(error)}) + status("rendering") + if failures: + raise RuntimeError("Some final videos failed; inspect status.json and per-sample render.log") + status("completed") + except Exception as error: + failures.append({"error": repr(error)}); status("failed"); raise + + +if __name__ == "__main__": + main() diff --git a/scripts/infer/run_multiturn_infer_suite.py b/scripts/infer/run_multiturn_infer_suite.py new file mode 100644 index 0000000000000000000000000000000000000000..8a210711097b0c8fa0bb0286c08d8711e456ddd8 --- /dev/null +++ b/scripts/infer/run_multiturn_infer_suite.py @@ -0,0 +1,700 @@ +#!/usr/bin/env python3 +"""Run HFD multi-turn evaluation cases through a dynamic GPU queue.""" + +from __future__ import annotations + +import hashlib +import json +import os +import re +import subprocess +import sys +import time +from collections import deque +from dataclasses import dataclass +from pathlib import Path +from typing import Any + + +PROJECT_ROOT = Path(__file__).resolve().parents[2] +DEFAULT_SUITE_V23 = PROJECT_ROOT / "scripts/infer/multiturn_auto_suite_v2.json" +DEFAULT_SUITE_V24 = PROJECT_ROOT / "scripts/infer/multiturn_auto_suite_v3.json" + + +@dataclass(frozen=True) +class InferenceCase: + name: str + interaction_mode: str + data_split: str + scenario_path: Path + scenario: dict[str, Any] + interaction_subtype: str = "" + + +def _truthy(value: str | None) -> bool: + return str(value or "").strip().lower() in {"1", "true", "yes", "on"} + + +def _motion_source_id(namespace: str, filename: str) -> str: + leaf = filename.replace("\\", "/").rstrip("/").rsplit("/", 1)[-1] + if leaf.lower().endswith(".csv"): + leaf = leaf[:-4] + return f"{namespace.strip().strip('/')}/{leaf}" + + +def _source_is_val(source_id: str, *, split_seed: int, val_ratio: float) -> bool: + digest = hashlib.sha1( + f"psi0-motion-source-split-v1\0{split_seed}\0{source_id}".encode() + ).digest() + threshold = int(float(val_ratio) * (1 << 64)) + return int.from_bytes(digest[:8], "big", signed=False) < threshold + + +def _load_json(path: Path) -> dict[str, Any]: + payload = json.loads(path.read_text(encoding="utf-8")) + if not isinstance(payload, dict): + raise ValueError(f"JSON root must be an object: {path}") + return payload + + +def validate_scenario( + path: Path, + *, + run_config: dict[str, Any], + expected_mode: str | None = None, + expected_split: str | None = None, + expected_subtype: str | None = None, +) -> InferenceCase: + payload = _load_json(path) + if payload.get("format") != "hfd_multiturn_auto_infer_scenario_v1": + raise ValueError(f"Unsupported multi-turn scenario: {path}") + turns = list(payload.get("turns", [])) + if not 2 <= len(turns) <= 4: + raise ValueError(f"Scenario must contain 2-4 turns, got {len(turns)}") + mode = str(payload.get("interaction_mode", "turn-taking")).strip().lower() + supported_modes = { + "turn-taking", + "interruption", + "emergency-stop", + "safe-hold", + "backchannel", + "hesitation", + "pause", + } + if mode not in supported_modes: + raise ValueError(f"Unsupported interaction_mode={mode!r}: {path}") + split = str(payload.get("data_split", "custom")).strip().lower() + if split not in {"train", "val", "custom"}: + raise ValueError(f"Unsupported data_split={split!r}: {path}") + if expected_mode is not None and mode != expected_mode: + raise ValueError(f"Suite/scenario mode mismatch: {expected_mode} != {mode}") + if expected_split is not None and split != expected_split: + raise ValueError(f"Suite/scenario split mismatch: {expected_split} != {split}") + subtype = str(payload.get("interaction_subtype", "")).strip().lower() + valid_subtypes = { + "hesitation": {"filled-pause", "self-repair"}, + "pause": {"silent-pause"}, + } + if mode in valid_subtypes: + if subtype not in valid_subtypes[mode]: + raise ValueError( + f"Invalid {mode} interaction_subtype={subtype!r}: {path}" + ) + elif subtype: + raise ValueError( + f"interaction_subtype is only valid for hesitation/pause: {path}" + ) + if expected_subtype is not None and subtype != expected_subtype: + raise ValueError( + f"Suite/scenario subtype mismatch: {expected_subtype} != {subtype}" + ) + + gaps = [int(value) for value in payload["inter_turn_background_steps_by_turn"]] + if len(gaps) != len(turns) - 1 or any(value < 0 for value in gaps): + raise ValueError(f"Invalid per-turn background gaps: {gaps}") + declared_gap_seconds = [ + float(value) + for value in payload.get("inter_turn_background_seconds_by_turn", []) + ] + if declared_gap_seconds: + expected_gap_seconds = [0.4 * value for value in gaps] + if len(declared_gap_seconds) != len(gaps) or any( + abs(observed - expected) > 1e-9 + for observed, expected in zip( + declared_gap_seconds, + expected_gap_seconds, + ) + ): + raise ValueError( + "Scenario gap seconds disagree with 0.4-second Qwen steps: " + f"{declared_gap_seconds} != {expected_gap_seconds}" + ) + if mode in {"interruption", "emergency-stop", "safe-hold"}: + gap_seconds = [0.4 * value for value in gaps] + if any(value < 2.0 or value > 6.0 for value in gap_seconds): + raise ValueError( + "Mid-motion command gaps must stay within 2-6 seconds: " + f"{gap_seconds}" + ) + if mode == "backchannel": + marked = [ + index + for index, turn in enumerate(turns) + if str(turn.get("backchannel_intent", "")).strip() + ] + if len(marked) != 1 or marked[0] == 0: + raise ValueError( + "Backchannel scenario needs exactly one acknowledgement after " + f"a real action command: indices={marked}" + ) + if str(turns[marked[0]]["backchannel_intent"]).strip() not in { + "continuer", + "assessment", + }: + raise ValueError("Unknown backchannel intent category") + voice_stems = { + Path(str(turn["audio_path"])).stem for turn in turns + } + if len(voice_stems) != 1: + raise ValueError( + "Backchannel scenario must keep one speaker/language identity: " + f"{sorted(voice_stems)}" + ) + onset_gap_seconds = 0.4 * gaps[marked[0] - 1] + if not 1.2 <= onset_gap_seconds <= 6.0: + raise ValueError( + "Backchannel must arrive during active motion: " + f"gap={onset_gap_seconds}s" + ) + + window_counts = [ + int(value) for value in payload.get("turn_audio_window_counts", []) + ] + expected_starts = [ + int(value) for value in payload.get("expected_turn_window_starts", []) + ] + if window_counts or expected_starts: + if len(window_counts) != len(turns) or len(expected_starts) != len(turns): + raise ValueError("Timeline audit needs one window count/start per turn") + derived_starts = [0] + for count, gap in zip(window_counts, gaps): + if count <= 0: + raise ValueError(f"Invalid audio window counts: {window_counts}") + derived_starts.append(derived_starts[-1] + count + gap) + if derived_starts != expected_starts: + raise ValueError( + f"Scenario gap timeline mismatch: {derived_starts} != {expected_starts}" + ) + + source_starts = [ + int(value) + for value in payload.get("source_turn_speech_start_frames_50fps", []) + ] + if source_starts: + if len(source_starts) != len(turns) or not expected_starts: + raise ValueError( + "Source timeline audit needs one source start and one expected " + "inference start per turn" + ) + relative_frames = [value - source_starts[0] for value in source_starts] + if any(value % 20 != 0 for value in relative_frames): + raise ValueError( + "Source speech starts are not aligned to the 20-frame Qwen grid: " + f"{source_starts}" + ) + source_relative_steps = [value // 20 for value in relative_frames] + if source_relative_steps != expected_starts: + raise ValueError( + "Scenario inference starts do not reproduce the source speech " + f"timeline: {source_relative_steps} != {expected_starts}" + ) + + data = dict(run_config.get("data", {})) + train = dict(run_config.get("train", {})) + active_semantics = int(train.get("full_duplex_semantics_version", 0)) + required_semantics = int(payload.get("required_semantics_version", 0)) + if active_semantics < required_semantics: + raise ValueError( + f"Scenario requires semantics {required_semantics}, run has {active_semantics}" + ) + if mode == "interruption" and ( + active_semantics < 20 or not bool(data.get("interruption_sessions", False)) + ): + raise ValueError("Interruption inference requires a semantics-v20 run") + if mode in {"emergency-stop", "safe-hold"}: + variants = { + "emergency-stop": int( + data.get("emergency_stop_session_variants", 0) + ), + "safe-hold": int(data.get("safe_hold_session_variants", 0)), + } + if ( + active_semantics < 21 + or not bool(data.get("independent_task_pools", False)) + or variants[mode] <= 0 + ): + raise ValueError( + f"{mode} inference requires an enabled semantics-v21 task pool" + ) + control_turns = [ + turn + for turn in turns + if str(turn.get("control_intent", "")).strip() + ] + if len(control_turns) != 1 or str( + control_turns[0].get("control_intent") + ).strip() != mode: + raise ValueError( + f"{mode} scenario needs exactly one matching control command" + ) + if mode == "backchannel": + if ( + active_semantics < 22 + or not bool(data.get("independent_task_pools", False)) + or int(data.get("backchannel_session_variants", 0)) <= 0 + ): + raise ValueError( + "Backchannel inference requires an enabled semantics-v22 pool" + ) + if mode in {"hesitation", "pause"}: + variants = { + "hesitation": int(data.get("hesitation_session_variants", 0)), + "pause": int(data.get("pause_session_variants", 0)), + } + if ( + active_semantics < 23 + or not bool(data.get("independent_task_pools", False)) + or variants[mode] <= 0 + ): + raise ValueError( + f"{mode} inference requires an enabled semantics-v23 task pool" + ) + event = dict(payload.get("disfluency_event", {})) + if str(event.get("subtype", "")).strip().lower() != subtype: + raise ValueError( + f"{mode} scenario has no matching disfluency_event: {path}" + ) + gap_steps = int(event.get("gap_steps", 0)) + if not 1 <= gap_steps <= 7: + raise ValueError( + f"{mode} gap must use 1-7 Qwen steps, got {gap_steps}" + ) + event_turn = int(event.get("turn_index", -1)) + if not 0 <= event_turn < len(turns): + raise ValueError(f"Invalid disfluency event turn: {event_turn}") + if str(turns[event_turn].get("disfluency_subtype", "")) != subtype: + raise ValueError( + "Scenario event turn does not carry the declared disfluency subtype" + ) + if active_semantics >= 24 and subtype == "self-repair": + repair_style = str(event.get("repair_style", "")).strip().lower() + if repair_style not in {"explicit-cue", "implicit-restart"}: + raise ValueError( + "Semantics-v24 self-repair scenario must declare " + f"repair_style, got {repair_style!r}" + ) + marker_id = str(event.get("marker_id", "")).strip() + if (repair_style == "explicit-cue") != bool(marker_id): + raise ValueError( + "Explicit self-repair requires a marker and implicit " + "restart must not carry one" + ) + + if split != "custom": + split_seed = int(data.get("split_seed", -1)) + val_ratio = float(data.get("val_ratio", -1.0)) + namespace = str(data.get("split_source_namespace", "")).strip().strip("/") + expected_namespace = str( + payload.get("split_source_namespace", "") + ).strip().strip("/") + if split_seed != int(payload.get("split_seed", -2)): + raise ValueError("Scenario/run split_seed mismatch") + if abs(val_ratio - float(payload.get("val_ratio", -2.0))) > 1e-12: + raise ValueError("Scenario/run val_ratio mismatch") + if not namespace or namespace != expected_namespace: + raise ValueError("Scenario/run split namespace mismatch") + for turn in turns: + filename = str(turn.get("source_motion_filename", "")).strip() + # Stop/hold utterances are control events, not new primitive + # motions. Their TTS path is audited above and deliberately has + # no train/val motion identity of its own. + if not filename and ( + str(turn.get("control_intent", "")).strip() + or str(turn.get("backchannel_intent", "")).strip() + ): + continue + if not filename: + raise ValueError("Split-tagged scenario turn has no motion filename") + observed = ( + "val" + if _source_is_val( + _motion_source_id(namespace, filename), + split_seed=split_seed, + val_ratio=val_ratio, + ) + else "train" + ) + if observed != split: + raise ValueError( + f"Scenario source split mismatch: {filename} is {observed}, " + f"scenario claims {split}" + ) + + for turn in turns: + audio_path = Path(str(turn["audio_path"])).expanduser().resolve() + if not audio_path.is_file(): + raise FileNotFoundError(f"Missing scenario audio: {audio_path}") + return InferenceCase( + name=str(payload["name"]), + interaction_mode=mode, + data_split=split, + scenario_path=path.resolve(), + scenario=payload, + interaction_subtype=subtype, + ) + + +def load_cases( + suite_path: Path, + *, + run_config: dict[str, Any], + single_scenario: Path | None = None, +) -> list[InferenceCase]: + if single_scenario is not None: + return [validate_scenario(single_scenario, run_config=run_config)] + suite = _load_json(suite_path) + if suite.get("format") not in { + "hfd_multiturn_auto_infer_suite_v2", + "hfd_multiturn_auto_infer_suite_v3", + }: + raise ValueError(f"Unsupported inference suite: {suite_path}") + data = dict(run_config.get("data", {})) + enabled_variants = { + "turn-taking": int(data.get("turn_taking_session_variants", 0)), + "interruption": int( + data.get("command_interruption_session_variants", 0) + ), + "emergency-stop": int( + data.get("emergency_stop_session_variants", 0) + ), + "safe-hold": int(data.get("safe_hold_session_variants", 0)), + "backchannel": int(data.get("backchannel_session_variants", 0)), + "hesitation": int(data.get("hesitation_session_variants", 0)), + "pause": int(data.get("pause_session_variants", 0)), + } + enabled_modes = { + mode for mode, variants in enabled_variants.items() if variants > 0 + } + cases: list[InferenceCase] = [] + for item in suite.get("cases", []): + item_mode = str(item["interaction_mode"]) + if item_mode not in enabled_modes: + continue + scenario_path = Path(str(item["scenario_file"])) + if not scenario_path.is_absolute(): + scenario_path = (suite_path.parent / scenario_path).resolve() + cases.append( + validate_scenario( + scenario_path, + run_config=run_config, + expected_mode=item_mode, + expected_split=str(item["data_split"]), + expected_subtype=str(item.get("interaction_subtype", "")), + ) + ) + identities = { + (case.interaction_mode, case.interaction_subtype, case.data_split) + for case in cases + } + expected = set() + for mode in enabled_modes: + subtypes = ( + ("filled-pause", "self-repair") + if mode == "hesitation" + else ("silent-pause",) + if mode == "pause" + else ("",) + ) + expected.update( + (mode, subtype, split) + for subtype in subtypes + for split in ("train", "val") + ) + if identities != expected: + raise ValueError( + "Suite must contain train/val cases for every enabled task: " + f"expected={expected} observed={identities}" + ) + return cases + + +def checkpoint_ready(path: Path) -> bool: + required = ( + path / "model.safetensors", + path / "hfd_fused_delta_manifest.json", + path / "full_duplex_source_boundary.json", + ) + if any(not item.is_file() or item.stat().st_size <= 0 for item in required): + return False + try: + marker = _load_json(required[-1]) + except Exception: + return False + return bool(marker.get("source_group_complete", False)) + + +def ready_checkpoint_steps(run_dir: Path) -> list[int]: + checkpoints = run_dir / "checkpoints" + found: list[int] = [] + for path in checkpoints.glob("ckpt_[0-9]*"): + match = re.fullmatch(r"ckpt_([0-9]+)", path.name) + if match and path.is_dir() and checkpoint_ready(path): + found.append(int(match.group(1))) + return sorted(found) + + +def visible_gpus() -> list[str]: + configured = os.environ.get("HFD_INFER_GPUS", "").strip() + if not configured: + configured = os.environ.get("CUDA_VISIBLE_DEVICES", "").strip() + if configured: + values = [value.strip() for value in configured.split(",") if value.strip()] + else: + result = subprocess.run( + ["nvidia-smi", "--query-gpu=index", "--format=csv,noheader"], + check=False, + text=True, + capture_output=True, + ) + values = [line.strip() for line in result.stdout.splitlines() if line.strip()] + if not values: + raise RuntimeError("No inference GPU is visible") + if len(values) != len(set(values)): + raise ValueError(f"Duplicate GPU identifiers: {values}") + return values + + +def _safe_component(value: str) -> str: + return re.sub(r"[^A-Za-z0-9_.-]+", "_", value) + + +def case_result_dir(run_dir: Path, case: InferenceCase, step: int) -> Path: + turns = list(case.scenario["turns"]) + # ``infer_render_auto.sh`` historically sanitizes ``echo "$stem"`` with + # ``tr -cs``. The terminating newline therefore becomes one trailing + # underscore. Reproduce that established on-disk name here rather than + # silently renaming or re-rendering every existing inference result. + audio_stem = _safe_component(Path(str(turns[0]["audio_path"])).stem + "\n") + sample_name = ( + f"00_{_safe_component(case.name)}_{_safe_component(audio_stem)}_" + f"session{len(turns)}turn" + ) + output_group = Path(case.interaction_mode) + if case.interaction_mode == "hesitation": + output_group /= ( + "filled" if case.interaction_subtype == "filled-pause" else "self-repair" + ) + return ( + run_dir + / output_group + / f"ckpt_{step}" + / case.data_split + / sample_name + ) + + +def case_settled(run_dir: Path, case: InferenceCase, step: int) -> bool: + result_dir = case_result_dir(run_dir, case, step) + return (result_dir / ".complete").is_file() or (result_dir / ".failed").is_file() + + +def case_failed(run_dir: Path, case: InferenceCase, step: int) -> bool: + return (case_result_dir(run_dir, case, step) / ".failed").is_file() + + +def case_environment( + run_dir: Path, + case: InferenceCase, + *, + step: int, + gpu: str, +) -> dict[str, str]: + payload = case.scenario + turns = list(payload["turns"]) + audio_paths = [str(Path(str(turn["audio_path"])).expanduser().resolve()) for turn in turns] + environment = os.environ.copy() + output_group = Path(case.interaction_mode) + if case.interaction_mode == "hesitation": + output_group /= ( + "filled" if case.interaction_subtype == "filled-pause" else "self-repair" + ) + environment.update( + { + "CUDA_VISIBLE_DEVICES": gpu, + "HFD_INFER_GPUS": gpu, + "HFD_INFER_DEVICE": "cuda:0", + "HFD_INFER_ONCE": "1", + "HFD_INFER_EXACT_CHECKPOINT_STEP": str(step), + "HFD_INFER_MIN_CHECKPOINT_STEP": str(step), + "HFD_INFER_OUTPUT_ROOT": str(run_dir / output_group), + "HFD_INFER_SAMPLE_PARENT": case.data_split, + "HFD_INFER_SAMPLE_TAG": case.name, + "HFD_INFER_SCENARIO_FILE": str(case.scenario_path), + "HFD_INFER_AUDIO_PATHS": audio_paths[0], + "HFD_INFER_ADDITIONAL_AUDIO_PATHS": ",".join(audio_paths[1:]), + "HFD_INFER_INITIAL_IDLE_STEPS": str(payload["initial_idle_steps"]), + "HFD_INFER_MOTION_FRAMES": str(payload["motion_frames_per_turn"]), + "HFD_INFER_INTER_TURN_BACKGROUND_STEPS_BY_TURN": ",".join( + str(int(value)) + for value in payload["inter_turn_background_steps_by_turn"] + ), + "HFD_INFER_TERMINAL_BACKGROUND_STEPS": str( + payload["terminal_background_steps"] + ), + } + ) + init_motion = str(payload.get("init_motion_dir", "")).strip() + if init_motion: + environment["HFD_INFER_INIT_MOTION_DIRS"] = init_motion + return environment + + +def run_dynamic_queue( + run_dir: Path, + cases: list[InferenceCase], + *, + step: int, + gpus: list[str], +) -> None: + pending = deque(case for case in cases if not case_settled(run_dir, case, step)) + available = deque(gpus) + running: dict[subprocess.Popen[bytes], tuple[InferenceCase, str]] = {} + failures: list[str] = [] + base_script = PROJECT_ROOT / "infer_render_auto.sh" + while pending or running: + while pending and available: + case = pending.popleft() + gpu = available.popleft() + print( + f"[suite] start ckpt_{step} {case.interaction_mode}/{case.data_split} " + f"on GPU {gpu}", + flush=True, + ) + process = subprocess.Popen( + ["bash", str(base_script), str(run_dir)], + cwd=PROJECT_ROOT, + env=case_environment(run_dir, case, step=step, gpu=gpu), + ) + running[process] = (case, gpu) + finished = [process for process in running if process.poll() is not None] + if not finished: + time.sleep(0.25) + continue + for process in finished: + case, gpu = running.pop(process) + available.append(gpu) + if process.returncode != 0: + failures.append( + f"{case.interaction_mode}/{case.data_split} " + f"exited {process.returncode}" + ) + elif case_failed(run_dir, case, step): + failures.append( + f"{case.interaction_mode}/{case.data_split} wrote .failed" + ) + elif not case_settled(run_dir, case, step): + failures.append( + f"{case.interaction_mode}/{case.data_split} wrote no marker" + ) + else: + print( + f"[suite] done ckpt_{step} " + f"{case.interaction_mode}/{case.data_split} on GPU {gpu}", + flush=True, + ) + if failures: + raise RuntimeError( + f"ckpt_{step} inference suite failed: " + "; ".join(failures) + ) + + +def main() -> None: + if len(sys.argv) != 2: + raise SystemExit(f"usage: {Path(sys.argv[0]).name} RUN_DIR") + run_dir = Path(sys.argv[1]).expanduser().resolve() + run_config_path = run_dir / "run_config.json" + if not run_config_path.is_file(): + raise SystemExit(f"Invalid HFD run directory: {run_dir}") + run_config = _load_json(run_config_path) + configured_suite = os.environ.get("HFD_MULTITURN_SUITE_FILE", "").strip() + active_semantics = int( + dict(run_config.get("train", {})).get( + "full_duplex_semantics_version", + 0, + ) + ) + suite_path = Path( + configured_suite + or (DEFAULT_SUITE_V24 if active_semantics >= 24 else DEFAULT_SUITE_V23) + ) + suite_path = suite_path.expanduser().resolve() + single_raw = os.environ.get("HFD_MULTITURN_SCENARIO_FILE", "").strip() + single_scenario = Path(single_raw).expanduser().resolve() if single_raw else None + cases = load_cases( + suite_path, + run_config=run_config, + single_scenario=single_scenario, + ) + gpus = visible_gpus() + check_interval = max(int(os.environ.get("HFD_INFER_CHECK_INTERVAL", "30")), 1) + infer_once = _truthy(os.environ.get("HFD_INFER_ONCE")) + exact_raw = os.environ.get("HFD_INFER_EXACT_CHECKPOINT_STEP", "").strip() + exact_step = int(exact_raw) if exact_raw else None + minimum_raw = os.environ.get("HFD_INFER_MIN_CHECKPOINT_STEP", "auto").strip() + initial_steps = ready_checkpoint_steps(run_dir) + if minimum_raw == "auto": + minimum_step = initial_steps[-1] if initial_steps else 0 + else: + minimum_step = int(minimum_raw) + print( + f"AUTO INFER SUITE: cases={len(cases)} GPUs={','.join(gpus)} " + f"min_step={minimum_step} exact_step={exact_step or 'none'}", + flush=True, + ) + + while True: + ready = ready_checkpoint_steps(run_dir) + if exact_step is not None: + candidates = [exact_step] if exact_step in ready else [] + else: + candidates = [step for step in ready if step >= minimum_step] + pending_steps = [ + step + for step in candidates + if any(not case_settled(run_dir, case, step) for case in cases) + ] + if pending_steps: + step = max(pending_steps) + run_dynamic_queue(run_dir, cases, step=step, gpus=gpus) + if infer_once or exact_step is not None: + break + else: + failed = [ + f"{case.interaction_mode}/{case.data_split}" + for step in candidates + for case in cases + if case_failed(run_dir, case, step) + ] + if failed: + raise RuntimeError( + "Existing inference failure marker(s) require inspection or " + f"cleanup before retry: {', '.join(failed)}" + ) + print(f"[{time.strftime('%F %T')}] no pending checkpoint", flush=True) + if infer_once or exact_step is not None: + break + time.sleep(check_interval) + + +if __name__ == "__main__": + main() diff --git a/scripts/infer/seed_audio_to_motion.py b/scripts/infer/seed_audio_to_motion.py new file mode 100644 index 0000000000000000000000000000000000000000..1a16629cff35b1d3e553ec4c11159b2095381cea --- /dev/null +++ b/scripts/infer/seed_audio_to_motion.py @@ -0,0 +1,530 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import argparse +import json +import math +import random +import re +import shutil +import subprocess +from pathlib import Path +from typing import List + +import numpy as np +import torch + +from psi.config.config import LaunchConfig +from psi.data.seed.motion_repr import quat_inv_wxyz, quat_mul_wxyz, recover_seed_abs_from_textop_action +from psi.data.seed.progress import build_progress_conditions +from psi.models.psi0 import Psi0Model +from psi.utils import parse_args_to_tyro_config, seed_everything + +from seed_text_to_motion import ( + TARGET_FPS, + build_model_states_from_abs, + finite_difference, + load_pose_from_motion_dir, + load_source_frame_count, + normalize_quaternion_rows, + quaternion_angular_velocity, + recover_seed_abs_from_textop_action_with_joint_delta, + resolve_ckpt_step, + sanitize_motion_name, + should_decode_textop_joint_from_delta, + write_csv, + write_metadata, +) + + +def cfg_get(obj, key: str, default=None): + if isinstance(obj, dict): + return obj.get(key, default) + return getattr(obj, key, default) + + +def audio_data_cfg(launch_config: LaunchConfig, key: str = "audio"): + return cfg_get(launch_config.data, key, launch_config.data) + + +def load_audio_array(video_path: Path, sampling_rate: int) -> np.ndarray: + cmd = [ + "ffmpeg", + "-v", + "error", + "-i", + str(video_path), + "-ac", + "1", + "-ar", + str(sampling_rate), + "-f", + "f32le", + "-", + ] + proc = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, check=False) + if proc.returncode != 0: + msg = proc.stderr.decode("utf-8", errors="ignore") + raise RuntimeError(f"ffmpeg failed to decode audio from {video_path}: {msg}") + audio = np.frombuffer(proc.stdout, dtype=np.float32).copy() + if audio.size == 0: + raise RuntimeError(f"No audio samples decoded from {video_path}") + return audio + + +def audio_segment(audio: np.ndarray, sampling_rate: int, start_sec: float, duration_sec: float) -> np.ndarray: + num_samples = max(1, int(round(duration_sec * sampling_rate))) + start_sample = int(round(start_sec * sampling_rate)) + end_sample = start_sample + num_samples + + segment = np.zeros((num_samples,), dtype=np.float32) + src_start = max(0, start_sample) + src_end = min(audio.shape[0], end_sample) + if src_end > src_start: + dst_start = src_start - start_sample + segment[dst_start : dst_start + (src_end - src_start)] = audio[src_start:src_end] + return segment + + +def read_start_hold_seconds(motion_dir: Path, align_start_hold: bool) -> float: + if not align_start_hold: + return 0.0 + info_path = motion_dir / "info.txt" + if not info_path.exists(): + return 0.0 + text = info_path.read_text(encoding="utf-8", errors="ignore") + match = re.search(r"start_hold_seconds:\s*([0-9.]+)", text) + return float(match.group(1)) if match else 0.0 + + +def resolve_motion_dir(root_dir: Path, motion_dir_arg: str) -> Path: + motion_dir = Path(motion_dir_arg).expanduser() + if motion_dir.is_absolute(): + return motion_dir + return root_dir / motion_dir + + +def resolve_video_path(video_dir: Path, motion_dir: Path, video_path_arg: str) -> Path: + if video_path_arg: + video_path = Path(video_path_arg).expanduser() + if video_path.is_absolute(): + return video_path + return video_dir / video_path + return video_dir / f"{motion_dir.name}.mp4" + + +def resolve_audio_path(audio_path: Path, extensions: str = ".wav,.flac,.mp3,.m4a,.aac,.ogg") -> Path: + if audio_path.is_file(): + return audio_path + if not audio_path.is_dir(): + return audio_path + audio_exts = tuple( + ext if ext.startswith(".") else f".{ext}" + for ext in [part.strip().lower() for part in str(extensions).split(",")] + if ext + ) + candidates = sorted(p for p in audio_path.iterdir() if p.is_file() and p.suffix.lower() in audio_exts) + if not candidates: + raise FileNotFoundError(f"No audio files found under {audio_path}") + return random.choice(candidates) + + +def normalize_action_for_rtc(maxmin, action: np.ndarray) -> np.ndarray: + action_min = np.asarray(maxmin.action_min, dtype=np.float32) + action_max = np.asarray(maxmin.action_max, dtype=np.float32) + ill_mask = np.abs(action_max - action_min) < 1e-4 * (np.abs(action_max) + np.abs(action_min) + 1e-8) + action_max = action_max.copy() + action_max[ill_mask] = 1.0 + action_norm = np.where(ill_mask, action, (action - action_min) / (action_max - action_min) * 2 - 1) + if bool(getattr(maxmin, "use_norm_mask", False)): + action_norm_masks = np.asarray(maxmin.action_norm_masks, dtype=bool) + action_norm = np.where(action_norm_masks, action_norm, action) + return np.clip(action_norm, -1.0, 1.0).astype(np.float32) + + +def rollout_audio_motion( + *, + model: Psi0Model, + maxmin, + odim: int, + device: str, + instruction: str, + audio: np.ndarray, + audio_sampling_rate: int, + audio_window_seconds: float, + audio_left_context_seconds: float, + full_audio: bool, + start_hold_seconds: float, + normalize_state: bool, + textop_actions: bool, + decode_textop_joint_from_delta: bool, + num_past_frames: int, + use_future_progress_embedding: bool, + enable_rtc: bool, + action_exec_horizon: int, + rtc_max_delay: int, + num_inference_steps: int, + target_frames: int, + init_abs_state: np.ndarray, +) -> tuple[np.ndarray, np.ndarray, int]: + horizon = int(model.action_horizon) + exec_horizon = int(action_exec_horizon) if enable_rtc else horizon + if exec_horizon <= 0 or exec_horizon > horizon: + raise ValueError(f"Invalid action_exec_horizon={exec_horizon}, action_horizon={horizon}") + rtc_delay = horizon - exec_horizon + if enable_rtc and rtc_delay > 0 and rtc_delay >= int(rtc_max_delay): + raise ValueError( + f"RTC requires action_horizon - action_exec_horizon < max_delay, " + f"got {horizon} - {exec_horizon} = {rtc_delay}, max_delay={rtc_max_delay}" + ) + if not textop_actions: + raise ValueError("This audio inference path is for the 39D TextOp action checkpoint.") + if target_frames <= 0: + target_frames = max(exec_horizon, int(round(audio.shape[0] / float(audio_sampling_rate) * TARGET_FPS))) + + num_chunks = int(math.ceil(float(target_frames) / float(exec_horizon))) + current_abs = np.asarray(init_abs_state, dtype=np.float32).copy() + state_horizon = int(num_past_frames) + 1 + state_history: List[np.ndarray] = [current_abs.copy() for _ in range(state_horizon)] + all_abs: List[np.ndarray] = [] + all_delta: List[np.ndarray] = [] + prev_actions_norm: np.ndarray | None = None + remaining_frames = int(target_frames) + + for _ in range(num_chunks): + frame_cursor = target_frames - remaining_frames + states = build_model_states_from_abs( + state_history[-state_horizon:], + textop_actions=textop_actions, + odim=odim, + ) + if normalize_state: + states = maxmin.normalize_state_func(states) + states_t = torch.from_numpy(states).unsqueeze(0).to(device) + future_progress_t = None + if use_future_progress_embedding: + _, future_progress = build_progress_conditions( + frame_id=frame_cursor, + total_frames=target_frames, + action_horizon=horizon, + ) + future_progress_t = torch.from_numpy(future_progress[None]).to(device) + + # 和训练一致:full_audio=True时每个chunk都看到整段音频embedding,不再给time/progress。 + if full_audio: + audio_clip = audio.astype(np.float32) + else: + audio_start_sec = ( + frame_cursor / float(TARGET_FPS) + - float(start_hold_seconds) + - float(audio_left_context_seconds) + ) + audio_clip = audio_segment(audio, audio_sampling_rate, audio_start_sec, audio_window_seconds) + + with torch.inference_mode(): + if enable_rtc and prev_actions_norm is not None and rtc_delay > 0: + prev_rtc = np.concatenate( + [ + prev_actions_norm[exec_horizon:, :], + np.zeros((exec_horizon, prev_actions_norm.shape[1]), dtype=np.float32), + ], + axis=0, + ) + prev_actions_t = torch.from_numpy(prev_rtc[None]).to(device) + pred_norm = model.predict_action_with_training_rtc_flow( + observations=None, + states=states_t, + instructions=[instruction], + num_inference_steps=num_inference_steps, + traj2ds=None, + prev_actions=prev_actions_t, + inference_delay=rtc_delay, + max_delay=int(rtc_max_delay), + audios=[audio_clip], + future_progress=future_progress_t, + )[0].float().cpu().numpy() + else: + pred_norm = model.predict_action( + observations=None, + states=states_t, + instructions=[instruction], + num_inference_steps=num_inference_steps, + traj2ds=None, + audios=[audio_clip], + future_progress=future_progress_t, + )[0].float().cpu().numpy() + + pred_norm_clipped = np.clip(np.asarray(pred_norm, dtype=np.float32), -1.0, 1.0) + chunk_pred = np.asarray(maxmin.denormalize(pred_norm_clipped), dtype=np.float32) + + if decode_textop_joint_from_delta: + chunk_abs = recover_seed_abs_from_textop_action_with_joint_delta(chunk_pred, current_abs) + else: + chunk_abs = recover_seed_abs_from_textop_action(chunk_pred, current_abs) + + prev_abs = np.concatenate([current_abs[None, :], chunk_abs[:-1]], axis=0) + joint_delta = chunk_abs[:, :29] - prev_abs[:, :29] + root_pos_delta = chunk_abs[:, 29:32] - prev_abs[:, 29:32] + root_quat_delta = quat_mul_wxyz(quat_inv_wxyz(prev_abs[:, 32:36]), chunk_abs[:, 32:36]) + chunk_delta = np.concatenate([joint_delta, root_pos_delta, root_quat_delta], axis=1) + + chunk_abs = normalize_quaternion_rows(chunk_abs.astype(np.float32)) + if decode_textop_joint_from_delta: + chunk_pred_for_rtc = chunk_pred.copy() + chunk_pred_for_rtc[:, :29] = chunk_abs[:, :29] + prev_actions_norm = normalize_action_for_rtc(maxmin, chunk_pred_for_rtc) + else: + prev_actions_norm = pred_norm_clipped.copy() + + take = min(exec_horizon, remaining_frames) + if take <= 0: + break + executed_abs = chunk_abs[:take] + executed_delta = chunk_delta[:take].astype(np.float32) + current_abs = executed_abs[-1].copy() + for frame_abs in executed_abs: + state_history.append(frame_abs.astype(np.float32).copy()) + state_history = state_history[-state_horizon:] + all_abs.append(executed_abs) + all_delta.append(executed_delta) + remaining_frames -= take + + pred_abs = np.concatenate(all_abs, axis=0)[:target_frames] + pred_delta = np.concatenate(all_delta, axis=0)[:target_frames] + pred_abs = normalize_quaternion_rows(pred_abs.astype(np.float32)) + return pred_abs, pred_delta, num_chunks + + +def write_audio_info( + path: Path, + *, + audio_config_key: str, + source_motion_dir: Path, + video_path: Path, + initial_pose_source: str, + frames: int, + textop_actions: bool, + num_past_frames: int, + enable_rtc: bool, + action_exec_horizon: int, + num_inference_steps: int, + audio_sampling_rate: int, + audio_window_seconds: float, + full_audio: bool, + instruction: str, +) -> None: + lines = [ + "Motion Information: seed_audio_to_motion_inference", + "===================================================", + "", + f"audio_config_key: {audio_config_key}", + f"source_motion_dir: {source_motion_dir}", + f"video_path: {video_path}", + f"instruction: {instruction}", + f"initial_pose_source: {initial_pose_source}", + f"target_fps: {TARGET_FPS}", + f"num_inference_steps: {num_inference_steps}", + f"trained_with_textop_actions: {textop_actions}", + f"num_past_frames: {num_past_frames}", + f"rtc_enabled: {enable_rtc}", + f"action_exec_horizon: {action_exec_horizon}", + f"audio_sampling_rate: {audio_sampling_rate}", + f"full_audio: {full_audio}", + f"audio_window_seconds: {audio_window_seconds}", + "output_root_quaternion: wxyz", + "output_joint_angles: 29D", + "output_root_position: xyz", + "", + f"frames: {frames}", + "", + ] + path.write_text("\n".join(lines), encoding="utf-8") + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description="Seed audio->motion offline inference exporter") + parser.add_argument("--run-dir", type=str, required=True) + parser.add_argument("--ckpt-step", type=str, default="latest", help="integer step, 'latest', or 'best'") + parser.add_argument("--output-dir", type=str, required=True) + parser.add_argument("--motion-dir", type=str, required=True) + parser.add_argument("--video-path", type=str, default="") + parser.add_argument( + "--audio-config-key", + type=str, + default="audio", + choices=("audio", "audio_command"), + help="Which launch_config.data audio section to use", + ) + parser.add_argument("--prompt", type=str, default="", help="Optional text instruction paired with the audio") + parser.add_argument("--num-output-frames", type=int, default=0, help="0 means source motion length") + parser.add_argument("--num-inference-steps", type=int, default=10) + parser.add_argument("--device", type=str, default="cuda:0") + parser.add_argument("--seed", type=int, default=42) + parser.add_argument("--export-delta", action="store_true", default=False) + parser.add_argument("--disable-rtc", action="store_true", default=False) + parser.add_argument("--no-align-start-hold", dest="align_start_hold", action="store_false") + parser.set_defaults(align_start_hold=True) + return parser.parse_args() + + +def main() -> None: + args = parse_args() + + run_dir = Path(args.run_dir) + output_dir = Path(args.output_dir) + output_dir.mkdir(parents=True, exist_ok=True) + + cfg_template: LaunchConfig = parse_args_to_tyro_config(run_dir / "argv.txt") # type: ignore + launch_config = cfg_template.model_validate_json((run_dir / "run_config.json").read_text()) + seed_everything(launch_config.seed or args.seed) + + ckpt_step = resolve_ckpt_step(run_dir, args.ckpt_step) + model = Psi0Model.from_pretrained(run_dir, ckpt_step, launch_config, device=args.device) + model.text_only = False + model.audio_only = True + model.video_only = False + model.to(args.device) + model.eval() + + audio_cfg = audio_data_cfg(launch_config, args.audio_config_key) + data_root = Path(cfg_get(audio_cfg, "root_dir")) + video_root = Path(cfg_get(audio_cfg, "video_dir", "")) + motion_dir = resolve_motion_dir(data_root, args.motion_dir) + video_path = resolve_audio_path( + resolve_video_path(video_root, motion_dir, args.video_path), + str(cfg_get(audio_cfg, "audio_file_extensions", ".wav,.flac,.mp3,.m4a,.aac,.ogg")), + ) + if not motion_dir.exists(): + raise FileNotFoundError(f"motion_dir does not exist: {motion_dir}") + if not video_path.exists(): + raise FileNotFoundError(f"video_path does not exist: {video_path}") + + maxmin = launch_config.data.transform.field + odim = int(launch_config.model.odim) + textop_actions = bool(getattr(launch_config.data.transform.repack, "textop_actions", False)) + decode_textop_joint_from_delta = textop_actions and should_decode_textop_joint_from_delta(launch_config) + num_past_frames = int(getattr(launch_config.data.transform.repack, "num_past_frames", 0)) + use_future_progress_embedding = bool(getattr(launch_config.model, "use_future_progress_embedding", False)) + enable_rtc = bool(getattr(launch_config.model, "rtc", False)) and (not args.disable_rtc) + action_exec_horizon = int(getattr(launch_config.model, "action_exec_horizon", int(model.action_horizon))) + rtc_max_delay = int(getattr(launch_config.model, "max_delay", 0)) + audio_sampling_rate = int(cfg_get(audio_cfg, "audio_sampling_rate", 16000)) + audio_window_seconds = float(cfg_get(audio_cfg, "audio_window_seconds", 1.0)) + audio_left_context_seconds = float(cfg_get(audio_cfg, "audio_left_context_seconds", 0.0)) + full_audio = bool(cfg_get(audio_cfg, "full_audio", False)) + + audio = load_audio_array(video_path, audio_sampling_rate) + instruction = args.prompt.strip() + x0 = load_pose_from_motion_dir(motion_dir) + source_frames = load_source_frame_count(motion_dir) + target_frames = int(args.num_output_frames) if args.num_output_frames > 0 else int(source_frames or 0) + start_hold_seconds = read_start_hold_seconds(motion_dir, bool(args.align_start_hold)) + + pred_abs, pred_delta, rollout_chunks = rollout_audio_motion( + model=model, + maxmin=maxmin, + odim=odim, + device=args.device, + instruction=instruction, + audio=audio, + audio_sampling_rate=audio_sampling_rate, + audio_window_seconds=audio_window_seconds, + audio_left_context_seconds=audio_left_context_seconds, + full_audio=full_audio, + start_hold_seconds=start_hold_seconds, + normalize_state=bool(getattr(maxmin, "normalize_state", False)), + textop_actions=textop_actions, + decode_textop_joint_from_delta=decode_textop_joint_from_delta, + num_past_frames=num_past_frames, + use_future_progress_embedding=use_future_progress_embedding, + enable_rtc=enable_rtc, + action_exec_horizon=action_exec_horizon, + rtc_max_delay=rtc_max_delay, + num_inference_steps=int(args.num_inference_steps), + target_frames=target_frames, + init_abs_state=x0, + ) + + motion_name = sanitize_motion_name(motion_dir.name) + sample_dir = output_dir / motion_name + sample_dir.mkdir(parents=True, exist_ok=True) + + joint_pos = pred_abs[:, :29] + body_pos = pred_abs[:, 29:32] + body_quat = pred_abs[:, 32:36] + + dt = 1.0 / float(TARGET_FPS) + joint_vel = finite_difference(joint_pos, dt) + body_lin_vel = finite_difference(body_pos, dt) + body_ang_vel = quaternion_angular_velocity(body_quat, dt) + + write_csv(sample_dir / "joint_pos.csv", joint_pos, [f"joint_{i}" for i in range(29)]) + write_csv(sample_dir / "joint_vel.csv", joint_vel, [f"joint_vel_{i}" for i in range(29)]) + write_csv(sample_dir / "body_pos.csv", body_pos, ["body_0_x", "body_0_y", "body_0_z"]) + write_csv(sample_dir / "body_quat.csv", body_quat, ["body_0_w", "body_0_x", "body_0_y", "body_0_z"]) + write_csv(sample_dir / "body_lin_vel.csv", body_lin_vel, ["body_0_vel_x", "body_0_vel_y", "body_0_vel_z"]) + write_csv(sample_dir / "body_ang_vel.csv", body_ang_vel, ["body_0_angvel_x", "body_0_angvel_y", "body_0_angvel_z"]) + np.save(sample_dir / "motion.npy", pred_abs) + + if args.export_delta: + write_csv(sample_dir / "joint_pos_delta.csv", pred_delta[:, :29], [f"joint_{i}" for i in range(29)]) + write_csv(sample_dir / "body_pos_delta.csv", pred_delta[:, 29:32], ["body_0_x", "body_0_y", "body_0_z"]) + write_csv(sample_dir / "body_quat_delta.csv", pred_delta[:, 32:36], ["body_0_w", "body_0_x", "body_0_y", "body_0_z"]) + np.save(sample_dir / "motion_delta.npy", pred_delta) + + write_metadata(sample_dir / "metadata.txt", motion_name, int(pred_abs.shape[0])) + write_audio_info( + sample_dir / "info.txt", + audio_config_key=str(args.audio_config_key), + source_motion_dir=motion_dir, + video_path=video_path, + initial_pose_source=str(motion_dir), + frames=int(pred_abs.shape[0]), + textop_actions=textop_actions, + num_past_frames=int(num_past_frames), + enable_rtc=enable_rtc, + action_exec_horizon=int(action_exec_horizon), + num_inference_steps=int(args.num_inference_steps), + audio_sampling_rate=int(audio_sampling_rate), + audio_window_seconds=float(audio_window_seconds), + full_audio=bool(full_audio), + instruction=instruction, + ) + + for src_name, dst_name in (("info.txt", "info_source.txt"), ("metadata.txt", "metadata_source.txt")): + src_path = motion_dir / src_name + if src_path.exists(): + shutil.copy2(src_path, sample_dir / dst_name) + + sample_meta = { + "run_dir": str(run_dir), + "ckpt_step": ckpt_step, + "audio_config_key": str(args.audio_config_key), + "instruction": instruction, + "source_motion_dir": str(motion_dir), + "video_path": str(video_path), + "output_dir": str(sample_dir), + "num_output_frames": int(pred_abs.shape[0]), + "rollout_chunks": int(rollout_chunks), + "textop_actions": textop_actions, + "decode_textop_joint_from_delta": decode_textop_joint_from_delta, + "num_past_frames": int(num_past_frames), + "enable_rtc": enable_rtc, + "action_exec_horizon": int(action_exec_horizon), + "audio_sampling_rate": int(audio_sampling_rate), + "full_audio": bool(full_audio), + "audio_window_seconds": float(audio_window_seconds), + "audio_left_context_seconds": float(audio_left_context_seconds), + "start_hold_seconds": float(start_hold_seconds), + } + (sample_dir / "infer_meta.json").write_text(json.dumps(sample_meta, ensure_ascii=False, indent=2), encoding="utf-8") + (output_dir / "manifest.json").write_text( + json.dumps({"samples": [sample_meta]}, ensure_ascii=False, indent=2), + encoding="utf-8", + ) + print(f"[OK] audio sample -> {sample_dir}") + print(f"[DONE] wrote manifest: {output_dir / 'manifest.json'}") + + +if __name__ == "__main__": + main() diff --git a/scripts/infer/seed_motion_teacher_reconstruct.py b/scripts/infer/seed_motion_teacher_reconstruct.py new file mode 100644 index 0000000000000000000000000000000000000000..e5f5a1edd0f41890ec7dd1d5be028d19ece9c48c --- /dev/null +++ b/scripts/infer/seed_motion_teacher_reconstruct.py @@ -0,0 +1,504 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import argparse +import json +import math +import shutil +from pathlib import Path +from typing import List + +import numpy as np +import pandas as pd +import torch +import torch.nn as nn +from safetensors import safe_open + +from psi.config.config import LaunchConfig +from psi.data.seed.motion_repr import ( + build_seed_textop_action, + quat_inv_wxyz, + quat_mul_wxyz, + recover_seed_abs_from_textop_action, +) +from psi.data.seed.progress import build_progress_conditions +from psi.models.motion_distill import MotionDistillPsi0Model +from psi.utils import parse_args_to_tyro_config, seed_everything + +from seed_text_to_motion import ( + TARGET_FPS, + build_model_states_from_abs, + finite_difference, + load_source_frame_count, + normalize_quaternion_rows, + quaternion_angular_velocity, + recover_seed_abs_from_textop_action_with_joint_delta, + resolve_ckpt_step, + sanitize_motion_name, + should_decode_textop_joint_from_delta, + write_csv, + write_metadata, +) + + +def resolve_checkpoint_dir(run_dir: Path, ckpt_step: int | str) -> Path: + if str(ckpt_step) in {"best", "best_ckpt"}: + ckpt_dir = run_dir / "checkpoints" / "best_ckpt" + else: + ckpt_dir = run_dir / "checkpoints" / f"ckpt_{ckpt_step}" + if not ckpt_dir.exists(): + root_ckpt_dir = run_dir / f"ckpt_{ckpt_step}" + if root_ckpt_dir.exists(): + ckpt_dir = root_ckpt_dir + if not ckpt_dir.exists(): + raise FileNotFoundError(f"Checkpoint dir does not exist: {ckpt_dir}") + return ckpt_dir + + +def load_selected_checkpoint_state(ckpt_dir: Path, prefixes: tuple[str, ...]) -> dict[str, torch.Tensor]: + model_path = ckpt_dir / "model.safetensors" + index_path = ckpt_dir / "model.safetensors.index.json" + shard_paths: list[Path] + + if index_path.exists(): + weight_map = json.loads(index_path.read_text()).get("weight_map", {}) + shard_paths = sorted( + {ckpt_dir / shard for key, shard in weight_map.items() if key.startswith(prefixes)} + ) + elif model_path.exists(): + shard_paths = [model_path] + else: + raise FileNotFoundError(f"No safetensors checkpoint found under {ckpt_dir}") + + state_dict: dict[str, torch.Tensor] = {} + for shard_path in shard_paths: + with safe_open(str(shard_path), framework="pt", device="cpu") as f: + for key in f.keys(): + if key.startswith(prefixes): + state_dict[key] = f.get_tensor(key) + return state_dict + + +def build_noise_scheduler(launch_config: LaunchConfig): + if launch_config.model.noise_scheduler == "ddpm": + from diffusers.schedulers.scheduling_ddim import DDIMScheduler + + return DDIMScheduler( + num_train_timesteps=launch_config.model.train_diffusion_steps, + beta_start=0.0001, + beta_end=0.02, + beta_schedule="squaredcos_cap_v2", + clip_sample=True, + set_alpha_to_one=True, + steps_offset=0, + prediction_type="epsilon", + ) + if launch_config.model.noise_scheduler == "flow": + from diffusers.schedulers.scheduling_flow_match_euler_discrete import FlowMatchEulerDiscreteScheduler + + return FlowMatchEulerDiscreteScheduler( + num_train_timesteps=launch_config.model.train_diffusion_steps, + ) + raise ValueError(f"Unsupported noise scheduler: {launch_config.model.noise_scheduler}") + + +def load_teacher_reconstruct_model( + run_dir: Path, + ckpt_step: int | str, + launch_config: LaunchConfig, + device: str, +) -> MotionDistillPsi0Model: + ckpt_dir = resolve_checkpoint_dir(run_dir, ckpt_step) + state_dict = load_selected_checkpoint_state( + ckpt_dir, + prefixes=("action_header.", "motion_encoder."), + ) + model = MotionDistillPsi0Model( + model_cfg=launch_config.model, + vlm_model=nn.Module(), + ) + action_state = { + key[len("action_header.") :]: value + for key, value in state_dict.items() + if key.startswith("action_header.") + } + motion_state = { + key[len("motion_encoder.") :]: value + for key, value in state_dict.items() + if key.startswith("motion_encoder.") + } + if not action_state: + raise ValueError(f"No action_header weights found in {ckpt_dir}") + if not motion_state: + raise ValueError(f"No motion_encoder weights found in {ckpt_dir}") + model.action_header.load_state_dict(action_state, strict=True) + model.motion_encoder.load_state_dict(motion_state, strict=True) + model.noise_scheduler = build_noise_scheduler(launch_config) + model.action_horizon = int(launch_config.model.action_chunk_size) + model.action_dim = int(launch_config.model.action_dim) + model.device = device + return model.to(device) + + +def load_motion_abs(motion_dir: Path) -> np.ndarray: + joint = pd.read_csv(motion_dir / "joint_pos.csv").to_numpy(dtype=np.float32) + body_pos = pd.read_csv(motion_dir / "body_pos.csv").to_numpy(dtype=np.float32) + body_quat = pd.read_csv(motion_dir / "body_quat.csv").to_numpy(dtype=np.float32) + if not (len(joint) == len(body_pos) == len(body_quat)): + raise ValueError(f"Motion csv length mismatch under {motion_dir}") + return np.concatenate([joint, body_pos, body_quat], axis=1).astype(np.float32) + + +def build_abs_action_chunk( + motion_abs: np.ndarray, + frame_id: int, + horizon: int, + sample_stride: int = 1, +) -> np.ndarray: + total_frames = int(motion_abs.shape[0]) + actions: List[np.ndarray] = [] + for step in range(horizon): + raw_index = frame_id + (step + 1) * sample_stride + actions.append(motion_abs[min(raw_index, total_frames - 1)]) + return np.stack(actions, axis=0).astype(np.float32) + + +def build_teacher_action_features( + actions_abs: np.ndarray, + current_abs: np.ndarray, + *, + textop_actions: bool, + action_dim: int, +) -> np.ndarray: + if textop_actions: + actions = build_seed_textop_action(actions_abs[:, :36], current_abs[:36]) + else: + actions = actions_abs[:, :action_dim] + + if actions.shape[-1] < action_dim: + pad = np.zeros((actions.shape[0], action_dim - actions.shape[-1]), dtype=np.float32) + actions = np.concatenate([actions, pad], axis=-1) + if actions.shape[-1] != action_dim: + raise ValueError(f"Teacher action dim mismatch: built {actions.shape[-1]}, expected {action_dim}") + return actions.astype(np.float32) + + +def normalize_action_only(maxmin, actions: np.ndarray) -> np.ndarray: + action_min = np.asarray(maxmin.action_min, dtype=np.float32) + action_max = np.asarray(maxmin.action_max, dtype=np.float32) + ill_mask = np.abs(action_max - action_min) < 1e-4 * ( + np.abs(action_max) + np.abs(action_min) + 1e-8 + ) + denom = action_max - action_min + denom[ill_mask] = 1.0 + actions_norm = np.where( + ill_mask, + actions, + (actions - action_min) / denom * 2 - 1, + ) + if bool(getattr(maxmin, "use_norm_mask", False)): + action_norm_masks = np.asarray(maxmin.action_norm_masks, dtype=bool) + actions_norm = np.where(action_norm_masks, actions_norm, actions) + return np.clip(actions_norm, -1.0, 1.0).astype(np.float32) + + +def decode_action_chunk( + chunk_pred: np.ndarray, + current_abs: np.ndarray, + *, + textop_actions: bool, + decode_textop_joint_from_delta: bool, +) -> tuple[np.ndarray, np.ndarray]: + if textop_actions: + if decode_textop_joint_from_delta: + chunk_abs = recover_seed_abs_from_textop_action_with_joint_delta(chunk_pred, current_abs) + else: + chunk_abs = recover_seed_abs_from_textop_action(chunk_pred, current_abs) + prev_abs = np.concatenate([current_abs[None, :], chunk_abs[:-1]], axis=0) + joint_delta = chunk_abs[:, :29] - prev_abs[:, :29] + root_pos_delta = chunk_abs[:, 29:32] - prev_abs[:, 29:32] + root_quat_delta = quat_mul_wxyz(quat_inv_wxyz(prev_abs[:, 32:36]), chunk_abs[:, 32:36]) + chunk_delta = np.concatenate([joint_delta, root_pos_delta, root_quat_delta], axis=1) + else: + chunk_abs = chunk_pred[:, :36] + first_delta = chunk_abs[0] - current_abs + if chunk_abs.shape[0] > 1: + rest_delta = chunk_abs[1:] - chunk_abs[:-1] + chunk_delta = np.concatenate([first_delta[None, :], rest_delta], axis=0) + else: + chunk_delta = first_delta[None, :] + + return normalize_quaternion_rows(chunk_abs.astype(np.float32)), chunk_delta.astype(np.float32) + + +def reconstruct_teacher_motion( + *, + model: MotionDistillPsi0Model, + maxmin, + motion_abs: np.ndarray, + odim: int, + action_dim: int, + device: str, + normalize_state: bool, + textop_actions: bool, + decode_textop_joint_from_delta: bool, + num_past_frames: int, + use_future_progress_embedding: bool, + num_inference_steps: int, + target_frames: int, +) -> tuple[np.ndarray, np.ndarray, dict[str, float]]: + horizon = int(model.action_horizon) + if target_frames <= 0: + target_frames = max(1, int(motion_abs.shape[0]) - 1) + + num_chunks = int(math.ceil(float(target_frames) / float(horizon))) + all_abs: List[np.ndarray] = [] + all_delta: List[np.ndarray] = [] + remaining_frames = int(target_frames) + + for chunk_id in range(num_chunks): + frame_id = chunk_id * horizon + current_abs = motion_abs[min(frame_id, motion_abs.shape[0] - 1)].astype(np.float32) + state_indices = [ + max(0, frame_id - (int(num_past_frames) - t)) + for t in range(int(num_past_frames) + 1) + ] + states = build_model_states_from_abs( + [motion_abs[min(i, motion_abs.shape[0] - 1)] for i in state_indices], + textop_actions=textop_actions, + odim=odim, + ) + if normalize_state: + states = maxmin.normalize_state_func(states) + + actions_abs = build_abs_action_chunk(motion_abs, frame_id, horizon) + teacher_actions = build_teacher_action_features( + actions_abs, + current_abs, + textop_actions=textop_actions, + action_dim=action_dim, + ) + teacher_actions_norm = normalize_action_only(maxmin, teacher_actions) + + with torch.inference_mode(): + states_t = torch.from_numpy(states).unsqueeze(0).to(device) + teacher_actions_t = torch.from_numpy(teacher_actions_norm).unsqueeze(0).to(device) + hidden_states = model.encode_motion_hidden(teacher_actions_t) + + future_progress_t = None + if use_future_progress_embedding: + _, future_progress = build_progress_conditions( + frame_id=frame_id, + total_frames=int(motion_abs.shape[0]), + action_horizon=horizon, + ) + future_progress_t = torch.from_numpy(future_progress[None]).to(device) + + action_samples = torch.randn(1, horizon, action_dim, device=device) + model.noise_scheduler.set_timesteps(num_inference_steps) + for timestep in model.noise_scheduler.timesteps: + batched_timestep = timestep.expand(1).to(device) + model_output = model.action_from_hidden( + hidden_states=hidden_states, + attention_mask=None, + action_samples=action_samples, + states=states_t, + timestep=batched_timestep, + traj2ds=None, + future_progress=future_progress_t, + return_dict=True, + ) + action_samples = model.noise_scheduler.step( + model_output=model_output.action, + timestep=timestep, + sample=action_samples, + ).prev_sample + + pred_norm = action_samples[0].float().cpu().numpy() + pred_norm_clipped = np.clip(np.asarray(pred_norm, dtype=np.float32), -1.0, 1.0) + chunk_pred = np.asarray(maxmin.denormalize(pred_norm_clipped), dtype=np.float32) + chunk_abs, chunk_delta = decode_action_chunk( + chunk_pred, + current_abs, + textop_actions=textop_actions, + decode_textop_joint_from_delta=decode_textop_joint_from_delta, + ) + + take = min(horizon, remaining_frames) + all_abs.append(chunk_abs[:take]) + all_delta.append(chunk_delta[:take]) + remaining_frames -= take + + pred_abs = np.concatenate(all_abs, axis=0)[:target_frames] + pred_delta = np.concatenate(all_delta, axis=0)[:target_frames] + pred_abs = normalize_quaternion_rows(pred_abs.astype(np.float32)) + + ref_abs = motion_abs[1 : 1 + pred_abs.shape[0], :36] + if ref_abs.shape[0] == pred_abs.shape[0]: + diff = pred_abs - ref_abs + metrics = { + "joint_mse": float(np.mean(diff[:, :29] ** 2)), + "root_pos_mse": float(np.mean(diff[:, 29:32] ** 2)), + "all_abs_mse": float(np.mean(diff ** 2)), + } + else: + metrics = {} + return pred_abs, pred_delta, metrics + + +def write_teacher_info( + path: Path, + *, + source_motion_dir: Path, + frames: int, + textop_actions: bool, + num_past_frames: int, + num_inference_steps: int, + metrics: dict[str, float], +) -> None: + lines = [ + "Motion Information: motion_distill_teacher_reconstruct", + "======================================================", + "", + f"source_motion_dir: {source_motion_dir}", + f"target_fps: {TARGET_FPS}", + f"num_inference_steps: {num_inference_steps}", + f"trained_with_textop_actions: {textop_actions}", + f"num_past_frames: {num_past_frames}", + "condition: motion_encoder(ground_truth_actions)", + "output_root_quaternion: wxyz", + "output_joint_angles: 29D", + "output_root_position: xyz", + "", + f"frames: {frames}", + "", + ] + for key, value in metrics.items(): + lines.append(f"{key}: {value:.8f}") + path.write_text("\n".join(lines), encoding="utf-8") + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description="Motion-distill teacher motion reconstruction exporter") + parser.add_argument("--run-dir", type=str, required=True) + parser.add_argument("--ckpt-step", type=str, default="best", help="integer step, 'latest', or 'best'") + parser.add_argument("--output-dir", type=str, required=True) + parser.add_argument("--motion-dir", type=str, required=True) + parser.add_argument("--num-output-frames", type=int, default=0, help="0 means source motion length minus one") + parser.add_argument("--num-inference-steps", type=int, default=10) + parser.add_argument("--device", type=str, default="cuda:0") + parser.add_argument("--seed", type=int, default=42) + parser.add_argument("--export-delta", action="store_true", default=False) + return parser.parse_args() + + +def main() -> None: + args = parse_args() + run_dir = Path(args.run_dir) + output_dir = Path(args.output_dir) + motion_dir = Path(args.motion_dir) + output_dir.mkdir(parents=True, exist_ok=True) + + cfg_template: LaunchConfig = parse_args_to_tyro_config(run_dir / "argv.txt") # type: ignore + launch_config = cfg_template.model_validate_json((run_dir / "run_config.json").read_text()) + seed_everything(launch_config.seed or args.seed) + + ckpt_step = resolve_ckpt_step(run_dir, args.ckpt_step) + model = load_teacher_reconstruct_model(run_dir, ckpt_step, launch_config, device=args.device) + model.eval() + + motion_abs = load_motion_abs(motion_dir) + source_frames = load_source_frame_count(motion_dir) or int(motion_abs.shape[0]) + target_frames = int(args.num_output_frames) if args.num_output_frames > 0 else max(1, source_frames - 1) + + maxmin = launch_config.data.transform.field + odim = int(launch_config.model.odim) + action_dim = int(getattr(launch_config.model, "action_dim", getattr(model, "action_dim", 39))) + textop_actions = bool(getattr(launch_config.data.transform.repack, "textop_actions", False)) + decode_textop_joint_from_delta = textop_actions and should_decode_textop_joint_from_delta(launch_config) + num_past_frames = int(getattr(launch_config.data.transform.repack, "num_past_frames", 0)) + use_future_progress_embedding = bool(getattr(launch_config.model, "use_future_progress_embedding", False)) + + pred_abs, pred_delta, metrics = reconstruct_teacher_motion( + model=model, + maxmin=maxmin, + motion_abs=motion_abs, + odim=odim, + action_dim=action_dim, + device=args.device, + normalize_state=bool(getattr(maxmin, "normalize_state", False)), + textop_actions=textop_actions, + decode_textop_joint_from_delta=decode_textop_joint_from_delta, + num_past_frames=num_past_frames, + use_future_progress_embedding=use_future_progress_embedding, + num_inference_steps=int(args.num_inference_steps), + target_frames=target_frames, + ) + + motion_name = sanitize_motion_name(motion_dir.name) + sample_dir = output_dir / motion_name + sample_dir.mkdir(parents=True, exist_ok=True) + + joint_pos = pred_abs[:, :29] + body_pos = pred_abs[:, 29:32] + body_quat = pred_abs[:, 32:36] + + dt = 1.0 / float(TARGET_FPS) + joint_vel = finite_difference(joint_pos, dt) + body_lin_vel = finite_difference(body_pos, dt) + body_ang_vel = quaternion_angular_velocity(body_quat, dt) + + write_csv(sample_dir / "joint_pos.csv", joint_pos, [f"joint_{i}" for i in range(29)]) + write_csv(sample_dir / "joint_vel.csv", joint_vel, [f"joint_vel_{i}" for i in range(29)]) + write_csv(sample_dir / "body_pos.csv", body_pos, ["body_0_x", "body_0_y", "body_0_z"]) + write_csv(sample_dir / "body_quat.csv", body_quat, ["body_0_w", "body_0_x", "body_0_y", "body_0_z"]) + write_csv(sample_dir / "body_lin_vel.csv", body_lin_vel, ["body_0_vel_x", "body_0_vel_y", "body_0_vel_z"]) + write_csv(sample_dir / "body_ang_vel.csv", body_ang_vel, ["body_0_angvel_x", "body_0_angvel_y", "body_0_angvel_z"]) + np.save(sample_dir / "motion.npy", pred_abs) + + if args.export_delta: + write_csv(sample_dir / "joint_pos_delta.csv", pred_delta[:, :29], [f"joint_{i}" for i in range(29)]) + write_csv(sample_dir / "body_pos_delta.csv", pred_delta[:, 29:32], ["body_0_x", "body_0_y", "body_0_z"]) + write_csv(sample_dir / "body_quat_delta.csv", pred_delta[:, 32:36], ["body_0_w", "body_0_x", "body_0_y", "body_0_z"]) + np.save(sample_dir / "motion_delta.npy", pred_delta) + + write_metadata(sample_dir / "metadata.txt", motion_name, int(pred_abs.shape[0])) + write_teacher_info( + sample_dir / "info.txt", + source_motion_dir=motion_dir, + frames=int(pred_abs.shape[0]), + textop_actions=textop_actions, + num_past_frames=num_past_frames, + num_inference_steps=int(args.num_inference_steps), + metrics=metrics, + ) + + for src_name, dst_name in (("info.txt", "info_source.txt"), ("metadata.txt", "metadata_source.txt")): + src_path = motion_dir / src_name + if src_path.exists(): + shutil.copy2(src_path, sample_dir / dst_name) + + sample_meta = { + "run_dir": str(run_dir), + "ckpt_step": ckpt_step, + "source_motion_dir": str(motion_dir), + "output_dir": str(sample_dir), + "num_output_frames": int(pred_abs.shape[0]), + "textop_actions": textop_actions, + "decode_textop_joint_from_delta": decode_textop_joint_from_delta, + "num_past_frames": int(num_past_frames), + "num_inference_steps": int(args.num_inference_steps), + "metrics": metrics, + } + (sample_dir / "infer_meta.json").write_text(json.dumps(sample_meta, ensure_ascii=False, indent=2), encoding="utf-8") + (output_dir / "manifest.json").write_text( + json.dumps({"samples": [sample_meta]}, ensure_ascii=False, indent=2), + encoding="utf-8", + ) + print(f"[OK] teacher reconstruction -> {sample_dir}") + if metrics: + print("[METRICS] " + json.dumps(metrics, ensure_ascii=False)) + + +if __name__ == "__main__": + main() diff --git a/scripts/infer/seed_text_to_motion.py b/scripts/infer/seed_text_to_motion.py new file mode 100644 index 0000000000000000000000000000000000000000..9d84a4b76aea19beca546d8f1c0b8565f0719f19 --- /dev/null +++ b/scripts/infer/seed_text_to_motion.py @@ -0,0 +1,873 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import argparse +import json +import math +import shutil +from pathlib import Path +from typing import List + +import numpy as np +import pandas as pd +import torch + +from psi.config.config import LaunchConfig +from psi.data.seed.motion_repr import ( + SEED_ABS_DIM, + SEED_TEXTOP_ACTION_DIM, + TEXTOP_JOINT_DELTA, + TEXTOP_ROOT_DELTA_LOCAL, + TEXTOP_ROOT_HEIGHT, + TEXTOP_ROOT_ROT6D_DELTA, + quat_inv_wxyz, + quat_mul_wxyz, + build_seed_textop_state, + recover_seed_abs_from_textop_action, + rot6d_delta_to_quat, + world_delta_from_local, +) +from psi.data.seed.progress import build_progress_conditions +from psi.data.seed.text_prompt import ( + DEFAULT_SEED_TEXT_FIELDS, + compose_seed_motion_prompt, + compose_seed_motion_prompt_from_row, +) +from psi.models.psi0 import Psi0Model +from psi.utils import parse_args_to_tyro_config, seed_everything + + +DEFAULT_TEXT_FIELDS: List[str] = list(DEFAULT_SEED_TEXT_FIELDS) + +TARGET_FPS = 50 + + +def compose_instruction(row: pd.Series, text_fields: List[str]) -> str: + return compose_seed_motion_prompt_from_row(row, text_fields) + + +def map_motion_dir(root_dir: Path, move_g1_path: str) -> Path: + rel = str(move_g1_path).strip().replace("\\", "/") + rel = rel.replace("g1/csv/", "g1/csv_sonic/") + rel = rel.replace("/csv/", "/csv_sonic/") + if rel.endswith(".csv"): + rel = rel[:-4] + + if rel.startswith("dataset/seed/"): + rel = rel[len("dataset/seed/") :] + if rel.startswith("seed/"): + rel = rel[len("seed/") :] + + p = Path(rel) + if p.is_absolute(): + return p + return root_dir / p + + +def sanitize_motion_name(name: str) -> str: + out = "".join(ch if (ch.isalnum() or ch in {"_", "-", "."}) else "_" for ch in name.strip()) + out = out.strip("._") + return out or "manual_prompt" + + +def normalize_quaternion_rows(motion_abs: np.ndarray) -> np.ndarray: + if motion_abs.shape[1] < 36: + return motion_abs + q = motion_abs[:, 32:36] + norm = np.linalg.norm(q, axis=1, keepdims=True) + norm = np.where(norm < 1e-8, 1.0, norm) + motion_abs[:, 32:36] = q / norm + return motion_abs + + +def quat_multiply(q1: np.ndarray, q2: np.ndarray) -> np.ndarray: + w1, x1, y1, z1 = q1 + w2, x2, y2, z2 = q2 + return np.array( + [ + w1 * w2 - x1 * x2 - y1 * y2 - z1 * z2, + w1 * x2 + x1 * w2 + y1 * z2 - z1 * y2, + w1 * y2 - x1 * z2 + y1 * w2 + z1 * x2, + w1 * z2 + x1 * y2 - y1 * x2 + z1 * w2, + ], + dtype=np.float64, + ) + + +def quat_conjugate(q: np.ndarray) -> np.ndarray: + return np.array([q[0], -q[1], -q[2], -q[3]], dtype=np.float64) + + +def quat_to_rotvec(q: np.ndarray) -> np.ndarray: + q = q / max(np.linalg.norm(q), 1e-12) + if q[0] < 0.0: + q = -q + vec = q[1:] + sin_half = np.linalg.norm(vec) + if sin_half < 1e-10: + return np.zeros(3, dtype=np.float64) + axis = vec / sin_half + angle = 2.0 * np.arctan2(sin_half, q[0]) + return axis * angle + + +def finite_difference(data: np.ndarray, dt: float) -> np.ndarray: + if data.shape[0] < 2: + return np.zeros_like(data, dtype=np.float64) + edge_order = 2 if data.shape[0] > 2 else 1 + return np.gradient(data, dt, axis=0, edge_order=edge_order) + + +def quaternion_angular_velocity(quat_wxyz: np.ndarray, dt: float) -> np.ndarray: + ang_vel = np.zeros((quat_wxyz.shape[0], 3), dtype=np.float64) + if quat_wxyz.shape[0] < 2: + return ang_vel + + for i in range(quat_wxyz.shape[0] - 1): + dq = quat_multiply(quat_wxyz[i + 1], quat_conjugate(quat_wxyz[i])) + ang_vel[i] = quat_to_rotvec(dq) / dt + + ang_vel[-1] = ang_vel[-2] + return ang_vel + + +def write_csv(path: Path, array: np.ndarray, headers: List[str]) -> None: + np.savetxt( + path, + np.asarray(array, dtype=np.float64), + delimiter=",", + header=",".join(headers), + comments="", + fmt="%.6f", + ) + + +def write_metadata(path: Path, motion_name: str, timesteps: int) -> None: + path.write_text( + "\n".join( + [ + f"Metadata for: {motion_name}", + "==============================", + "", + "Body part indexes:", + "[0]", + "", + f"Total timesteps: {timesteps}", + "", + ] + ), + encoding="utf-8", + ) + + +def write_info( + path: Path, + *, + source_path: str, + initial_pose_source: str, + frames: int, + use_delta_actions: bool, + root_delta_actions: bool, + textop_actions: bool, + num_past_frames: int, + enable_rtc: bool, + action_exec_horizon: int, + num_inference_steps: int, +) -> None: + lines = [ + "Motion Information: seed_text_to_motion_inference", + "==================================================", + "", + f"source_path: {source_path}", + f"initial_pose_source: {initial_pose_source}", + f"target_fps: {TARGET_FPS}", + f"num_inference_steps: {num_inference_steps}", + f"trained_with_delta_actions: {use_delta_actions}", + f"trained_with_root_delta_actions: {root_delta_actions}", + f"trained_with_textop_actions: {textop_actions}", + f"num_past_frames: {num_past_frames}", + f"rtc_enabled: {enable_rtc}", + f"action_exec_horizon: {action_exec_horizon}", + "output_root_quaternion: wxyz", + "output_joint_angles: 29D", + "output_root_position: xyz", + "", + f"frames: {frames}", + "", + ] + path.write_text("\n".join(lines), encoding="utf-8") + + +def load_pose_from_motion_dir(motion_dir: Path) -> np.ndarray: + joint = pd.read_csv(motion_dir / "joint_pos.csv").to_numpy(dtype=np.float32) + body_pos = pd.read_csv(motion_dir / "body_pos.csv").to_numpy(dtype=np.float32) + body_quat = pd.read_csv(motion_dir / "body_quat.csv").to_numpy(dtype=np.float32) + return np.concatenate([joint[0], body_pos[0], body_quat[0]], axis=0).astype(np.float32) + + +def load_pose_from_npy(path: Path) -> np.ndarray: + pose = np.load(path).astype(np.float32) + if pose.ndim == 2: + pose = pose[0] + if pose.shape != (36,): + raise ValueError(f"Expected init pose shape (36,) or (T,36), got {pose.shape} from {path}") + return pose.astype(np.float32) + + +def resolve_motion_dir(data_root: Path, motion_dir_arg: str) -> Path: + motion_dir = Path(motion_dir_arg).expanduser() + if motion_dir.is_absolute(): + return motion_dir + return data_root / motion_dir + + +def load_initial_pose(root_dir: Path, row: pd.Series, restore_from_data: bool, strict: bool = False) -> np.ndarray: + if not restore_from_data: + if strict: + # root-delta也必须有真实首帧,否则root轨迹会从zero累加跑偏。 + raise ValueError("Delta/root-delta inference requires a real initial pose. Do not use --no-restore-from-data.") + return np.zeros((36,), dtype=np.float32) + + try: + motion_dir = map_motion_dir(root_dir, str(row["move_g1_path"])) + return load_pose_from_motion_dir(motion_dir) + except Exception as exc: + if strict: + raise RuntimeError(f"Failed to load initial pose from metadata row: {exc}") from exc + return np.zeros((36,), dtype=np.float32) + + +def load_source_frame_count(source_motion_dir: Path) -> int | None: + try: + joint = pd.read_csv(source_motion_dir / "joint_pos.csv") + frames = int(len(joint)) + if frames > 0: + return frames + except Exception: + pass + return None + + +def build_model_states_from_abs( + state_history_abs: List[np.ndarray], + *, + textop_actions: bool, + odim: int, +) -> np.ndarray: + states_abs = np.stack(state_history_abs, axis=0).astype(np.float32) + states = build_seed_textop_state(states_abs[:, :SEED_ABS_DIM]) if textop_actions else states_abs + if states.shape[-1] < odim: + pad = np.zeros((states.shape[0], odim - states.shape[-1]), dtype=np.float32) + states = np.concatenate([states, pad], axis=-1) + if states.shape[-1] != odim: + raise ValueError(f"Infer state dim mismatch: built {states.shape[-1]}D, model odim={odim}") + return states.astype(np.float32) + + +def should_decode_textop_joint_from_delta(launch_config: LaunchConfig) -> bool: + if SEED_TEXTOP_ACTION_DIM == 39: + return False + group_weight = getattr(getattr(launch_config, "train", None), "loss_group_weight", None) + if group_weight is None: + return False + joint_abs_weight = float(getattr(group_weight, "joint_abs", 1.0)) + joint_delta_weight = float(getattr(group_weight, "joint_delta", 1.0)) + return joint_abs_weight == 0.0 and joint_delta_weight != 0.0 + + +def recover_seed_abs_from_textop_action_with_joint_delta( + features: np.ndarray, + current_abs: np.ndarray, +) -> np.ndarray: + # breakpoint() + # 当joint_abs不参与训练时,推理侧也改成用joint_delta递推关节,避免使用未监督的joint_abs通道。 + features = np.asarray(features, dtype=np.float32) + current_abs = np.asarray(current_abs, dtype=np.float32).copy() + if features.shape[-1] != SEED_TEXTOP_ACTION_DIM: + raise ValueError(f"Expected textop action dim {SEED_TEXTOP_ACTION_DIM}, got {features.shape[-1]}") + if current_abs.shape[-1] != SEED_ABS_DIM: + raise ValueError(f"Expected current_abs dim {SEED_ABS_DIM}, got {current_abs.shape[-1]}") + if TEXTOP_JOINT_DELTA.stop == TEXTOP_JOINT_DELTA.start: + return recover_seed_abs_from_textop_action(features, current_abs) + + frames = [] + prev_abs = current_abs.copy() + for feat in features: + joint_delta = feat[TEXTOP_JOINT_DELTA] + root_delta_local = feat[TEXTOP_ROOT_DELTA_LOCAL] + root_rot_delta = feat[TEXTOP_ROOT_ROT6D_DELTA] + root_height = feat[TEXTOP_ROOT_HEIGHT][0] + + joint_abs = prev_abs[:29] + joint_delta + root_pos = prev_abs[29:32] + world_delta_from_local(prev_abs[32:36], root_delta_local) + root_pos[2] = root_height + rel_quat = rot6d_delta_to_quat(root_rot_delta) + root_quat = quat_mul_wxyz(prev_abs[32:36], rel_quat) + + frame = np.concatenate([joint_abs, root_pos.astype(np.float32), root_quat.astype(np.float32)], axis=0) + frame = frame.astype(np.float32) + frames.append(frame) + prev_abs = frame + + return np.stack(frames, axis=0).astype(np.float32) + + +def resolve_metadata_csv(data_root: Path, launch_config: LaunchConfig, metadata_csv_arg: str) -> Path | None: + if metadata_csv_arg: + metadata_csv = Path(metadata_csv_arg) + elif hasattr(launch_config.data, "metadata_csv"): + metadata_csv = Path(launch_config.data.metadata_csv) + elif hasattr(launch_config.data, "text"): + text_cfg = launch_config.data.text + metadata_csv_value = text_cfg.get("metadata_csv") if isinstance(text_cfg, dict) else getattr(text_cfg, "metadata_csv", None) + if metadata_csv_value is None: + return None + metadata_csv = Path(metadata_csv_value) + else: + return None + if not metadata_csv.is_absolute(): + metadata_csv = data_root / metadata_csv + return metadata_csv + + +def resolve_text_data_root(launch_config: LaunchConfig) -> Path: + if hasattr(launch_config.data, "root_dir"): + return Path(launch_config.data.root_dir) + if hasattr(launch_config.data, "text"): + text_cfg = launch_config.data.text + root_dir = text_cfg.get("root_dir") if isinstance(text_cfg, dict) else getattr(text_cfg, "root_dir", None) + if root_dir: + return Path(root_dir) + raise AttributeError("run config has neither data.root_dir nor data.text.root_dir") + + +def load_prompt_initial_pose( + *, + args: argparse.Namespace, + data_root: Path, + metadata_csv: Path | None, + use_delta_actions: bool, + root_delta_actions: bool, + textop_actions: bool, +) -> tuple[np.ndarray, str]: + if args.init_pose_npy: + # 允许prompt模式显式指定36D初始姿态,避免delta模型从zero起步 + init_path = Path(args.init_pose_npy).expanduser() + if not init_path.is_absolute(): + cwd_path = Path.cwd() / init_path + data_path = data_root / init_path + init_path = cwd_path if cwd_path.exists() else data_path + return load_pose_from_npy(init_path), str(init_path) + + if args.init_motion_dir: + # 允许prompt模式显式指定SEED动作目录,用该动作第一帧作为初始姿态 + motion_dir = resolve_motion_dir(data_root, args.init_motion_dir) + return load_pose_from_motion_dir(motion_dir), str(motion_dir) + + if args.restore_from_data: + # 默认从metadata的start-index动作取真实首帧,不再静默使用zero + if metadata_csv is None: + raise ValueError( + "Prompt inference needs an initial pose. Use --init-motion-dir or --init-pose-npy " + "when the run config has no metadata_csv." + ) + prompt_metadata = pd.read_csv(metadata_csv) + if len(prompt_metadata) == 0: + raise ValueError(f"Empty metadata csv: {metadata_csv}") + ref_index = min(max(0, int(args.start_index)), len(prompt_metadata) - 1) + ref_row = prompt_metadata.iloc[ref_index] + motion_dir = map_motion_dir(data_root, str(ref_row["move_g1_path"])) + return load_pose_from_motion_dir(motion_dir), f"{motion_dir} (metadata index {ref_index})" + + # hybrid root-delta prompt推理同样需要真实初始root。 + if use_delta_actions or root_delta_actions or textop_actions: + raise ValueError( + "Delta/root-delta/TextOp prompt inference requires a real initial pose. " + "Use --init-motion-dir, --init-pose-npy, or keep --restore-from-data." + ) + + return np.zeros((36,), dtype=np.float32), "zero" + + +def rollout_motion( + *, + model: Psi0Model, + maxmin, + odim: int, + device: str, + instruction: str, + text_only: bool, + normalize_state: bool, + use_delta_actions: bool, + root_delta_actions: bool, + textop_actions: bool, + decode_textop_joint_from_delta: bool, + num_past_frames: int, + use_future_progress_embedding: bool, + enable_rtc: bool, + action_exec_horizon: int, + rtc_max_delay: int, + num_inference_steps: int, + target_frames: int, + init_abs_state: np.ndarray, +) -> tuple[np.ndarray, np.ndarray, int]: + horizon = int(model.action_horizon) + exec_horizon = int(action_exec_horizon) if enable_rtc else horizon + if exec_horizon <= 0 or exec_horizon > horizon: + raise ValueError(f"Invalid action_exec_horizon={exec_horizon}, action_horizon={horizon}") + rtc_delay = horizon - exec_horizon + if enable_rtc and rtc_delay > 0 and rtc_delay >= int(rtc_max_delay): + raise ValueError( + f"RTC requires action_horizon - action_exec_horizon < max_delay, " + f"got {horizon} - {exec_horizon} = {rtc_delay}, max_delay={rtc_max_delay}" + ) + if target_frames <= 0: + target_frames = exec_horizon + # RTC推理按exec_horizon滚动;普通推理仍按完整action_horizon滚动 + # 原逻辑:num_chunks = int(math.ceil(float(target_frames) / float(horizon))) + num_chunks = int(math.ceil(float(target_frames) / float(exec_horizon))) + + current_abs = np.asarray(init_abs_state, dtype=np.float32).copy() + # 推理时维护和训练一致的多帧state历史,首个chunk用初始姿态重复填充。 + state_horizon = int(num_past_frames) + 1 + state_history: List[np.ndarray] = [current_abs.copy() for _ in range(state_horizon)] + all_abs: List[np.ndarray] = [] + all_delta: List[np.ndarray] = [] + prev_actions_norm: np.ndarray | None = None + remaining_frames = int(target_frames) + + for _ in range(num_chunks): + generated_frames = int(target_frames - remaining_frames) + # 使用最近To帧state作为输入;保持shape(B, To, Ds)。 + # 原逻辑: + # states = current_abs.reshape(1, odim).astype(np.float32) + states = build_model_states_from_abs( + state_history[-state_horizon:], + textop_actions=textop_actions, + odim=odim, + ) + if normalize_state: + states = maxmin.normalize_state_func(states) + states_t = torch.from_numpy(states).unsqueeze(0).to(device) + future_progress_t = None + if use_future_progress_embedding: + _, future_progress = build_progress_conditions( + frame_id=generated_frames, + total_frames=target_frames, + action_horizon=horizon, + ) + future_progress_t = torch.from_numpy(future_progress[None]).to(device) + + with torch.inference_mode(): + if enable_rtc and prev_actions_norm is not None and rtc_delay > 0: + # RTC推理把上一chunk尚未执行的尾部动作作为下一chunk的prefix条件 + prev_rtc = np.concatenate( + [ + prev_actions_norm[exec_horizon:, :], + np.zeros((exec_horizon, prev_actions_norm.shape[1]), dtype=np.float32), + ], + axis=0, + ) + prev_actions_t = torch.from_numpy(prev_rtc[None]).to(device) + pred_norm = model.predict_action_with_training_rtc_flow( + observations=None if text_only else [[]], + states=states_t, + instructions=[instruction], + num_inference_steps=num_inference_steps, + traj2ds=None, + prev_actions=prev_actions_t, + inference_delay=rtc_delay, + max_delay=int(rtc_max_delay), + future_progress=future_progress_t, + )[0].float().cpu().numpy() + else: + pred_norm = model.predict_action( + observations=None if text_only else [[]], + states=states_t, + instructions=[instruction], + num_inference_steps=num_inference_steps, + traj2ds=None, + future_progress=future_progress_t, + )[0].float().cpu().numpy() + + # 训练侧ActionStateTransform会把normalized action裁剪到[-1,1],推理反归一化前也要保持同一范围。 + # 原逻辑:chunk_pred = np.asarray(maxmin.denormalize(pred_norm), dtype=np.float32) + pred_norm_clipped = np.clip(np.asarray(pred_norm, dtype=np.float32), -1.0, 1.0) + chunk_pred = np.asarray(maxmin.denormalize(pred_norm_clipped), dtype=np.float32) + + # TextOp训练目标先恢复到36D绝对姿态;root旋转delta在helper里按相对旋转6D解码。 + if textop_actions: + if decode_textop_joint_from_delta: + chunk_abs = recover_seed_abs_from_textop_action_with_joint_delta(chunk_pred, current_abs) + else: + chunk_abs = recover_seed_abs_from_textop_action(chunk_pred, current_abs) + prev_abs = np.concatenate([current_abs[None, :], chunk_abs[:-1]], axis=0) + joint_delta = chunk_abs[:, :29] - prev_abs[:, :29] + root_pos_delta = chunk_abs[:, 29:32] - prev_abs[:, 29:32] + root_quat_delta = quat_mul_wxyz(quat_inv_wxyz(prev_abs[:, 32:36]), chunk_abs[:, 32:36]) + chunk_delta = np.concatenate([joint_delta, root_pos_delta, root_quat_delta], axis=1) + elif use_delta_actions: + # delta action需要从current_abs累加成绝对姿态,支持多chunk滚动 + # 原逻辑:把chunk_pred直接当绝对姿态,和delta训练目标不一致 + chunk_delta = chunk_pred + chunk_abs = np.cumsum(chunk_delta, axis=0) + current_abs[None, :] + elif root_delta_actions: + # hybrid action:关节直接用绝对预测,root 7维从当前姿态逐帧累加。 + chunk_abs = chunk_pred.copy() + root_abs = current_abs[29:36].astype(np.float32).copy() + root_abs_seq = [] + for root_delta in chunk_pred[:, 29:36]: + root_abs = root_abs + root_delta + quat_norm = np.linalg.norm(root_abs[3:7]) + if quat_norm > 1e-8: + root_abs[3:7] = root_abs[3:7] / quat_norm + root_abs_seq.append(root_abs.copy()) + chunk_abs[:, 29:36] = np.stack(root_abs_seq, axis=0) + + first_joint_delta = chunk_abs[0, :29] - current_abs[:29] + if chunk_abs.shape[0] > 1: + rest_joint_delta = chunk_abs[1:, :29] - chunk_abs[:-1, :29] + joint_delta = np.concatenate([first_joint_delta[None, :], rest_joint_delta], axis=0) + else: + joint_delta = first_joint_delta[None, :] + chunk_delta = np.concatenate([joint_delta, chunk_pred[:, 29:36]], axis=1) + else: + chunk_abs = chunk_pred + first_delta = chunk_abs[0] - current_abs + if chunk_abs.shape[0] > 1: + rest_delta = chunk_abs[1:] - chunk_abs[:-1] + chunk_delta = np.concatenate([first_delta[None, :], rest_delta], axis=0) + else: + chunk_delta = first_delta[None, :] + + chunk_abs = normalize_quaternion_rows(chunk_abs.astype(np.float32)) + # TextOp且joint_abs未监督时,RTC prefix不能继续使用模型预测的joint_abs通道。 + # 原逻辑:prev_actions_norm = np.asarray(pred_norm, dtype=np.float32).copy() + # 这里用joint_delta还原后的关节绝对值回填0:29,再重新normalize,避免坏joint_abs污染下一chunk。 + if textop_actions and decode_textop_joint_from_delta: + chunk_pred_for_rtc = chunk_pred.copy() + chunk_pred_for_rtc[:, :29] = chunk_abs[:, :29] + # 这里不能直接调用maxmin.normalize,因为当前配置normalize_state=True时会要求data里有states。 + action_min = np.asarray(maxmin.action_min, dtype=np.float32) + action_max = np.asarray(maxmin.action_max, dtype=np.float32) + ill_mask = np.abs(action_max - action_min) < 1e-4 * (np.abs(action_max) + np.abs(action_min) + 1e-8) + action_max = action_max.copy() + action_max[ill_mask] = 1.0 + chunk_pred_for_rtc_norm = np.where( + ill_mask, + chunk_pred_for_rtc, + (chunk_pred_for_rtc - action_min) / (action_max - action_min) * 2 - 1, + ) + if bool(getattr(maxmin, "use_norm_mask", False)): + action_norm_masks = np.asarray(maxmin.action_norm_masks, dtype=bool) + chunk_pred_for_rtc_norm = np.where(action_norm_masks, chunk_pred_for_rtc_norm, chunk_pred_for_rtc) + prev_actions_norm = np.clip(chunk_pred_for_rtc_norm, -1, 1).astype(np.float32).copy() + else: + # RTC prefix也使用裁剪后的normalized action,避免把越界预测作为clean prefix条件。 + # 原逻辑:prev_actions_norm = np.asarray(pred_norm, dtype=np.float32).copy() + prev_actions_norm = pred_norm_clipped.copy() + # RTC只导出本轮实际执行的前exec_horizon帧;普通模式exec_horizon等于horizon + take = min(exec_horizon, remaining_frames) + if take <= 0: + break + executed_abs = chunk_abs[:take] + executed_delta = chunk_delta[:take].astype(np.float32) + current_abs = executed_abs[-1].copy() + # 每执行一步都推进state历史,下一chunk看到的是真实rollout历史而不是单帧。 + for frame_abs in executed_abs: + state_history.append(frame_abs.astype(np.float32).copy()) + state_history = state_history[-state_horizon:] + all_abs.append(executed_abs) + all_delta.append(executed_delta) + remaining_frames -= take + + pred_abs = np.concatenate(all_abs, axis=0)[:target_frames] + pred_delta = np.concatenate(all_delta, axis=0)[:target_frames] + pred_abs = normalize_quaternion_rows(pred_abs.astype(np.float32)) + return pred_abs, pred_delta, num_chunks + + +def resolve_ckpt_step(run_dir: Path, ckpt_step: str) -> int | str: + if ckpt_step == "best": + # 支持--ckpt-step best,直接读取checkpoints/best_ckpt目录。 + best_ckpt = run_dir / "checkpoints" / "best_ckpt" + if not best_ckpt.exists(): + raise RuntimeError(f"No best_ckpt found under {run_dir / 'checkpoints'}") + return "best_ckpt" + + if ckpt_step != "latest": + return int(ckpt_step) + + ckpt_root = run_dir / "checkpoints" + candidates = [] + for p in ckpt_root.iterdir(): + if p.is_dir() and p.name.startswith("ckpt_"): + try: + candidates.append(int(p.name.split("_")[-1])) + except ValueError: + continue + if not candidates: + raise RuntimeError(f"No ckpt_* found under {ckpt_root}") + return max(candidates) + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description="Seed text->motion offline inference exporter") + parser.add_argument("--run-dir", type=str, required=True) + parser.add_argument("--ckpt-step", type=str, default="latest", help="integer step, 'latest', or 'best'") + parser.add_argument("--output-dir", type=str, required=True) + parser.add_argument("--num-samples", type=int, default=1) + parser.add_argument("--start-index", type=int, default=0) + parser.add_argument("--num-inference-steps", type=int, default=10) + parser.add_argument("--device", type=str, default="cuda:0") + parser.add_argument("--metadata-csv", type=str, default="") + parser.add_argument("--prompt", type=str, default="", help="Direct text prompt for manual inference") + parser.add_argument("--prompt-motion-name", type=str, default="manual_prompt") + parser.add_argument("--init-motion-dir", type=str, default="", help="SEED csv_sonic motion dir used for prompt initial pose") + parser.add_argument("--init-pose-npy", type=str, default="", help="36D pose npy, or Tx36 motion npy, used for prompt initial pose") + parser.add_argument("--num-output-frames", type=int, default=0, help="0 means auto length") + parser.add_argument("--seed", type=int, default=42) + parser.add_argument("--restore-from-data", action="store_true", default=True) + parser.add_argument("--no-restore-from-data", dest="restore_from_data", action="store_false") + parser.add_argument("--export-delta", action="store_true", default=False) + parser.add_argument("--disable-rtc", action="store_true", default=False) + parser.add_argument( + "--force-text-only", + action="store_true", + default=False, + help="Use the same checkpoint but force text-only VLM inputs, even if run_config is audio-only.", + ) + return parser.parse_args() + + +def main() -> None: + args = parse_args() + + run_dir = Path(args.run_dir) + output_dir = Path(args.output_dir) + output_dir.mkdir(parents=True, exist_ok=True) + + cfg_template: LaunchConfig = parse_args_to_tyro_config(run_dir / "argv.txt") # type: ignore + launch_config = cfg_template.model_validate_json((run_dir / "run_config.json").read_text()) + + seed_everything(launch_config.seed or args.seed) + + # 找到指定ckpt + ckpt_step = resolve_ckpt_step(run_dir, args.ckpt_step) + model = Psi0Model.from_pretrained(run_dir, ckpt_step, launch_config, device=args.device) + if args.force_text_only: + model.text_only = True + model.audio_only = False + model.video_only = False + model.to(args.device) + model.eval() + + data_root = resolve_text_data_root(launch_config) + prompt_mode = bool(args.prompt.strip()) + use_delta_actions = bool(getattr(launch_config.data.transform.repack, "use_delta_actions", False)) + # run_config里记录hybrid action模式:关节绝对,root delta。 + root_delta_actions = bool(getattr(launch_config.data.transform.repack, "root_delta_actions", False)) + # TextOp模式下训练目标是39D action/state;推理端用同一个decoder恢复为36D绝对动作。 + textop_actions = bool(getattr(launch_config.data.transform.repack, "textop_actions", False)) + decode_textop_joint_from_delta = textop_actions and should_decode_textop_joint_from_delta(launch_config) + # 从run_config读取SEED多帧state配置;旧checkpoint默认退化为单帧state。 + num_past_frames = int(getattr(launch_config.data.transform.repack, "num_past_frames", 0)) + # run_config里model.rtc=True时默认使用RTC滚动推理;可用--disable-rtc关闭 + enable_rtc = bool(getattr(launch_config.model, "rtc", False)) and (not args.disable_rtc) + use_future_progress_embedding = bool(getattr(launch_config.model, "use_future_progress_embedding", False)) + action_exec_horizon = int(getattr(launch_config.model, "action_exec_horizon", int(model.action_horizon))) + rtc_max_delay = int(getattr(launch_config.model, "max_delay", 0)) + metadata_csv = resolve_metadata_csv(data_root, launch_config, args.metadata_csv) + + metadata = None + if not prompt_mode: + metadata = pd.read_csv(metadata_csv) + start = max(0, args.start_index) + end = min(len(metadata), start + max(0, args.num_samples)) + if start >= end: + raise ValueError(f"Empty sampling range: start={start}, end={end}, metadata_size={len(metadata)}") + sample_indices = list(range(start, end)) + else: + sample_indices = [0] + + text_only = bool(args.force_text_only) or bool(getattr(launch_config.data.transform.model, "text_only", False)) + maxmin = launch_config.data.transform.field + odim = int(launch_config.model.odim) + action_dim = int(getattr(launch_config.model, "action_dim", getattr(model, "action_dim", SEED_TEXTOP_ACTION_DIM))) + if textop_actions and (action_dim != SEED_TEXTOP_ACTION_DIM or odim != SEED_TEXTOP_ACTION_DIM): + raise ValueError( + f"TextOp infer/train dim mismatch: action_dim={action_dim}, odim={odim}, expected={SEED_TEXTOP_ACTION_DIM}" + ) + + manifest = { + "run_dir": str(run_dir), + "ckpt_step": ckpt_step, + "metadata_csv": str(metadata_csv) if metadata_csv is not None else "", + "mode": "prompt" if prompt_mode else "metadata", + "text_only": text_only, + "action_dim": int(action_dim), + "state_dim": int(odim), + "use_delta_actions": use_delta_actions, + "root_delta_actions": root_delta_actions, + "textop_actions": textop_actions, + "decode_textop_joint_from_delta": decode_textop_joint_from_delta, + "num_past_frames": int(num_past_frames), + "use_future_progress_embedding": bool(use_future_progress_embedding), + "enable_rtc": enable_rtc, + "action_exec_horizon": int(action_exec_horizon), + "rtc_max_delay": int(rtc_max_delay), + "num_inference_steps": int(args.num_inference_steps), + "num_output_frames": int(args.num_output_frames), + "export_delta": bool(args.export_delta), + "samples": [], + } + + for idx in sample_indices: + if prompt_mode: + raw_instruction = args.prompt.strip() + instruction = compose_seed_motion_prompt(raw_instruction) + source_motion_dir = Path("manual_prompt") + # prompt模式下delta推理不能再用zero初始姿态,默认从metadata的start-index动作取真实首帧 + # 原逻辑:x0 = np.zeros((36,), dtype=np.float32) + x0, initial_pose_source = load_prompt_initial_pose( + args=args, + data_root=data_root, + metadata_csv=metadata_csv, + use_delta_actions=use_delta_actions, + root_delta_actions=root_delta_actions, + textop_actions=textop_actions, + ) + auto_frames = max(int(model.action_horizon), 300) + target_frames = int(args.num_output_frames) if args.num_output_frames > 0 else auto_frames + motion_name = sanitize_motion_name(args.prompt_motion_name) + sample_dir = output_dir / "g1" / "csv_sonic" / "manual" / motion_name + else: + assert metadata is not None + row = metadata.iloc[idx] + raw_instruction = "" + instruction = compose_instruction(row, DEFAULT_TEXT_FIELDS) + source_motion_dir = map_motion_dir(data_root, str(row.get("move_g1_path", ""))) + # delta推理加载初始姿态失败时直接报错,避免静默退回zero + x0 = load_initial_pose( + data_root, + row, + restore_from_data=args.restore_from_data, + strict=(use_delta_actions or root_delta_actions or textop_actions), + ) + initial_pose_source = str(source_motion_dir) if args.restore_from_data else "zero" + source_frames = load_source_frame_count(source_motion_dir) + if args.num_output_frames > 0: + target_frames = int(args.num_output_frames) + elif source_frames is not None: + target_frames = source_frames + else: + target_frames = int(model.action_horizon) + # Output folder keeps the same relative hierarchy as csv_sonic + try: + rel_dir = source_motion_dir.relative_to(data_root) + except Exception: + rel_dir = Path(f"sample_{idx:04d}") + sample_dir = output_dir / rel_dir + motion_name = source_motion_dir.name if source_motion_dir.name else f"sample_{idx:04d}" + + pred_abs, pred_delta, rollout_chunks = rollout_motion( + model=model, + maxmin=maxmin, + odim=odim, + device=args.device, + instruction=instruction, + text_only=text_only, + normalize_state=bool(getattr(maxmin, "normalize_state", False)), + use_delta_actions=use_delta_actions, + root_delta_actions=root_delta_actions, + textop_actions=textop_actions, + decode_textop_joint_from_delta=decode_textop_joint_from_delta, + num_past_frames=num_past_frames, + use_future_progress_embedding=use_future_progress_embedding, + enable_rtc=enable_rtc, + action_exec_horizon=action_exec_horizon, + rtc_max_delay=rtc_max_delay, + num_inference_steps=int(args.num_inference_steps), + target_frames=target_frames, + init_abs_state=x0, + ) + + sample_dir.mkdir(parents=True, exist_ok=True) + + joint_pos = pred_abs[:, :29] + body_pos = pred_abs[:, 29:32] + body_quat = pred_abs[:, 32:36] + + dt = 1.0 / float(TARGET_FPS) + joint_vel = finite_difference(joint_pos, dt) + body_lin_vel = finite_difference(body_pos, dt) + body_ang_vel = quaternion_angular_velocity(body_quat, dt) + + # Main outputs: match csv_sonic naming/headers + write_csv(sample_dir / "joint_pos.csv", joint_pos, [f"joint_{i}" for i in range(29)]) + write_csv(sample_dir / "joint_vel.csv", joint_vel, [f"joint_vel_{i}" for i in range(29)]) + write_csv(sample_dir / "body_pos.csv", body_pos, ["body_0_x", "body_0_y", "body_0_z"]) + write_csv(sample_dir / "body_quat.csv", body_quat, ["body_0_w", "body_0_x", "body_0_y", "body_0_z"]) + write_csv(sample_dir / "body_lin_vel.csv", body_lin_vel, ["body_0_vel_x", "body_0_vel_y", "body_0_vel_z"]) + write_csv(sample_dir / "body_ang_vel.csv", body_ang_vel, ["body_0_angvel_x", "body_0_angvel_y", "body_0_angvel_z"]) + + np.save(sample_dir / "motion.npy", pred_abs) + + # Optional delta outputs + if args.export_delta: + write_csv(sample_dir / "joint_pos_delta.csv", pred_delta[:, :29], [f"joint_{i}" for i in range(29)]) + write_csv(sample_dir / "body_pos_delta.csv", pred_delta[:, 29:32], ["body_0_x", "body_0_y", "body_0_z"]) + write_csv(sample_dir / "body_quat_delta.csv", pred_delta[:, 32:36], ["body_0_w", "body_0_x", "body_0_y", "body_0_z"]) + np.save(sample_dir / "motion_delta.npy", pred_delta) + + write_metadata(sample_dir / "metadata.txt", motion_name, int(pred_abs.shape[0])) + write_info( + sample_dir / "info.txt", + source_path=str(source_motion_dir), + initial_pose_source=initial_pose_source, + frames=int(pred_abs.shape[0]), + use_delta_actions=use_delta_actions, + root_delta_actions=root_delta_actions, + textop_actions=textop_actions, + num_past_frames=int(num_past_frames), + enable_rtc=enable_rtc, + action_exec_horizon=int(action_exec_horizon), + num_inference_steps=int(args.num_inference_steps), + ) + + # Keep source references if existing + if not prompt_mode: + src_info = source_motion_dir / "info.txt" + src_meta = source_motion_dir / "metadata.txt" + if src_info.exists(): + shutil.copy2(src_info, sample_dir / "info_source.txt") + if src_meta.exists(): + shutil.copy2(src_meta, sample_dir / "metadata_source.txt") + + sample_meta = { + "sample_index": int(idx), + "instruction": instruction, + "raw_instruction": raw_instruction, + "move_g1_path": "" if prompt_mode else str(metadata.iloc[idx].get("move_g1_path", "")), + "source_motion_dir": str(source_motion_dir), + "initial_pose_source": initial_pose_source, + "num_past_frames": int(num_past_frames), + "root_delta_actions": root_delta_actions, + "textop_actions": textop_actions, + "decode_textop_joint_from_delta": decode_textop_joint_from_delta, + "enable_rtc": enable_rtc, + "action_exec_horizon": int(action_exec_horizon), + "output_dir": str(sample_dir), + "num_output_frames": int(pred_abs.shape[0]), + "rollout_chunks": int(rollout_chunks), + } + (sample_dir / "infer_meta.json").write_text(json.dumps(sample_meta, ensure_ascii=False, indent=2)) + manifest["samples"].append(sample_meta) + print(f"[OK] sample={idx} -> {sample_dir}") + + (output_dir / "manifest.json").write_text(json.dumps(manifest, ensure_ascii=False, indent=2)) + print(f"[DONE] wrote manifest: {output_dir / 'manifest.json'}") + + +if __name__ == "__main__": + main() diff --git a/scripts/infer/seed_video_to_motion.py b/scripts/infer/seed_video_to_motion.py new file mode 100644 index 0000000000000000000000000000000000000000..f60ffcb023f57fc27d20b3ca17aa981d4b426826 --- /dev/null +++ b/scripts/infer/seed_video_to_motion.py @@ -0,0 +1,548 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import argparse +import json +import math +import shutil +from pathlib import Path +from typing import List + +import numpy as np +import torch + +from psi.config.config import LaunchConfig +from psi.data.seed.motion_repr import quat_inv_wxyz, quat_mul_wxyz, recover_seed_abs_from_textop_action +from psi.data.seed.progress import build_progress_conditions +from psi.data.seed.seed_video_motion_dataset import ( + _probe_video, + load_video_array, + load_video_clip_array, + load_video_embed_cache, + video_embed_cache_path, +) +from psi.models.motion_distill import MotionDistillPsi0Model +from psi.models.psi0 import Psi0Model +from psi.utils import parse_args_to_tyro_config, seed_everything + +from seed_audio_to_motion import resolve_motion_dir, resolve_video_path +from seed_text_to_motion import ( + TARGET_FPS, + build_model_states_from_abs, + finite_difference, + load_pose_from_motion_dir, + load_source_frame_count, + normalize_quaternion_rows, + quaternion_angular_velocity, + recover_seed_abs_from_textop_action_with_joint_delta, + resolve_ckpt_step, + sanitize_motion_name, + should_decode_textop_joint_from_delta, + write_csv, + write_metadata, +) + + +def cfg_get(obj, key: str, default=None): + if isinstance(obj, dict): + return obj.get(key, default) + return getattr(obj, key, default) + + +def video_data_cfg(launch_config: LaunchConfig, config_key: str = "video"): + return cfg_get(launch_config.data, config_key, cfg_get(launch_config.data, "video", launch_config.data)) + + +def rollout_video_motion( + *, + model: Psi0Model, + maxmin, + odim: int, + device: str, + video: np.ndarray | None, + video_embed: dict | None, + video_path: Path, + instruction: str, + video_size_pixels: int, + video_sample_fps: float, + video_decoder: str, + video_pixel_cache_dtype: str, + video_clip_mode: bool, + video_clip_motion_fps: float, + normalize_state: bool, + textop_actions: bool, + decode_textop_joint_from_delta: bool, + num_past_frames: int, + use_future_progress_embedding: bool, + enable_rtc: bool, + action_exec_horizon: int, + rtc_max_delay: int, + num_inference_steps: int, + target_frames: int, + init_abs_state: np.ndarray, +) -> tuple[np.ndarray, np.ndarray, int]: + horizon = int(model.action_horizon) + exec_horizon = int(action_exec_horizon) if enable_rtc else horizon + if exec_horizon <= 0 or exec_horizon > horizon: + raise ValueError(f"Invalid action_exec_horizon={exec_horizon}, action_horizon={horizon}") + rtc_delay = horizon - exec_horizon + if enable_rtc and rtc_delay > 0 and rtc_delay >= int(rtc_max_delay): + raise ValueError( + f"RTC requires action_horizon - action_exec_horizon < max_delay, " + f"got {horizon} - {exec_horizon} = {rtc_delay}, max_delay={rtc_max_delay}" + ) + if not textop_actions: + raise ValueError("This video inference path is for the 39D TextOp action checkpoint.") + if target_frames <= 0: + target_frames = exec_horizon + + num_chunks = int(math.ceil(float(target_frames) / float(exec_horizon))) + current_abs = np.asarray(init_abs_state, dtype=np.float32).copy() + state_horizon = int(num_past_frames) + 1 + state_history: List[np.ndarray] = [current_abs.copy() for _ in range(state_horizon)] + all_abs: List[np.ndarray] = [] + all_delta: List[np.ndarray] = [] + prev_actions_norm: np.ndarray | None = None + remaining_frames = int(target_frames) + + for _ in range(num_chunks): + if video_clip_mode: + chunk_index = len(all_abs) + video_clip_start_sec = float(chunk_index * exec_horizon) / float(video_clip_motion_fps) + video_clip_duration_sec = float(horizon) / float(video_clip_motion_fps) + chunk_video, _, _ = load_video_clip_array( + video_path, + start_sec=video_clip_start_sec, + duration_sec=video_clip_duration_sec, + video_sample_fps=video_sample_fps, + video_size_pixels=video_size_pixels, + video_decoder=video_decoder, + ) + chunk_video_embed = None + else: + chunk_video = video + chunk_video_embed = video_embed + + frame_cursor = target_frames - remaining_frames + states = build_model_states_from_abs( + state_history[-state_horizon:], + textop_actions=textop_actions, + odim=odim, + ) + if normalize_state: + states = maxmin.normalize_state_func(states) + states_t = torch.from_numpy(states).unsqueeze(0).to(device) + future_progress_t = None + if use_future_progress_embedding: + if video_clip_mode: + # Training video_clip samples are cropped to one local horizon: + # state frame 0 plus action frames 1..horizon. Keep progress local + # at inference so action-token positions match the training distribution. + progress_frame_id = 0 + progress_total_frames = horizon + 1 + else: + progress_frame_id = frame_cursor + progress_total_frames = target_frames + _, future_progress = build_progress_conditions( + frame_id=progress_frame_id, + total_frames=progress_total_frames, + action_horizon=horizon, + ) + future_progress_t = torch.from_numpy(future_progress[None]).to(device) + + with torch.inference_mode(): + if enable_rtc and prev_actions_norm is not None and rtc_delay > 0: + prev_rtc = np.concatenate( + [ + prev_actions_norm[exec_horizon:, :], + np.zeros((exec_horizon, prev_actions_norm.shape[1]), dtype=np.float32), + ], + axis=0, + ) + prev_actions_t = torch.from_numpy(prev_rtc[None]).to(device) + pred_norm = model.predict_action_with_training_rtc_flow( + observations=None, + states=states_t, + instructions=[instruction], + num_inference_steps=num_inference_steps, + traj2ds=None, + prev_actions=prev_actions_t, + inference_delay=rtc_delay, + max_delay=int(rtc_max_delay), + videos=[chunk_video] if chunk_video_embed is None else None, + video_embeds=[chunk_video_embed] if chunk_video_embed is not None else None, + video_paths=[str(video_path)], + video_size_pixels=int(video_size_pixels), + video_sample_fps=float(video_sample_fps), + video_pixel_cache_dtype=video_pixel_cache_dtype, + future_progress=future_progress_t, + )[0].float().cpu().numpy() + else: + pred_norm = model.predict_action( + observations=None, + states=states_t, + instructions=[instruction], + num_inference_steps=num_inference_steps, + traj2ds=None, + videos=[chunk_video] if chunk_video_embed is None else None, + video_embeds=[chunk_video_embed] if chunk_video_embed is not None else None, + video_paths=[str(video_path)], + video_size_pixels=int(video_size_pixels), + video_sample_fps=float(video_sample_fps), + video_pixel_cache_dtype=video_pixel_cache_dtype, + future_progress=future_progress_t, + )[0].float().cpu().numpy() + + pred_norm_clipped = np.clip(np.asarray(pred_norm, dtype=np.float32), -1.0, 1.0) + chunk_pred = np.asarray(maxmin.denormalize(pred_norm_clipped), dtype=np.float32) + + if decode_textop_joint_from_delta: + chunk_abs = recover_seed_abs_from_textop_action_with_joint_delta(chunk_pred, current_abs) + else: + chunk_abs = recover_seed_abs_from_textop_action(chunk_pred, current_abs) + + prev_abs = np.concatenate([current_abs[None, :], chunk_abs[:-1]], axis=0) + joint_delta = chunk_abs[:, :29] - prev_abs[:, :29] + root_pos_delta = chunk_abs[:, 29:32] - prev_abs[:, 29:32] + root_quat_delta = quat_mul_wxyz(quat_inv_wxyz(prev_abs[:, 32:36]), chunk_abs[:, 32:36]) + chunk_delta = np.concatenate([joint_delta, root_pos_delta, root_quat_delta], axis=1) + + chunk_abs = normalize_quaternion_rows(chunk_abs.astype(np.float32)) + if decode_textop_joint_from_delta: + chunk_pred_for_rtc = chunk_pred.copy() + chunk_pred_for_rtc[:, :29] = chunk_abs[:, :29] + from seed_audio_to_motion import normalize_action_for_rtc + prev_actions_norm = normalize_action_for_rtc(maxmin, chunk_pred_for_rtc) + else: + prev_actions_norm = pred_norm_clipped.copy() + + take = min(exec_horizon, remaining_frames) + if take <= 0: + break + executed_abs = chunk_abs[:take] + executed_delta = chunk_delta[:take].astype(np.float32) + current_abs = executed_abs[-1].copy() + for frame_abs in executed_abs: + state_history.append(frame_abs.astype(np.float32).copy()) + state_history = state_history[-state_horizon:] + all_abs.append(executed_abs) + all_delta.append(executed_delta) + remaining_frames -= take + + pred_abs = np.concatenate(all_abs, axis=0)[:target_frames] + pred_delta = np.concatenate(all_delta, axis=0)[:target_frames] + pred_abs = normalize_quaternion_rows(pred_abs.astype(np.float32)) + return pred_abs, pred_delta, num_chunks + + +def write_video_info( + path: Path, + *, + video_decoder: str, + source_motion_dir: Path, + video_path: Path, + instruction: str, + initial_pose_source: str, + frames: int, + textop_actions: bool, + num_past_frames: int, + enable_rtc: bool, + action_exec_horizon: int, + num_inference_steps: int, + video_sample_fps: float, + video_size_pixels: int, + video_pixel_cache_dtype: str, + video_embed_path: str, + source_video_fps: float, + video_duration_sec: float, + full_video: bool, + video_config_key: str, + video_clip_motion_fps: float, +) -> None: + lines = [ + "Motion Information: seed_video_to_motion_inference", + "===================================================", + "", + f"video_decoder: {video_decoder}", + f"video_config_key: {video_config_key}", + f"source_motion_dir: {source_motion_dir}", + f"video_path: {video_path}", + f"instruction: {instruction}", + f"initial_pose_source: {initial_pose_source}", + f"target_fps: {TARGET_FPS}", + f"num_inference_steps: {num_inference_steps}", + f"trained_with_textop_actions: {textop_actions}", + f"num_past_frames: {num_past_frames}", + f"rtc_enabled: {enable_rtc}", + f"action_exec_horizon: {action_exec_horizon}", + f"video_sample_fps: {video_sample_fps}", + f"video_size_pixels: {video_size_pixels}", + f"video_pixel_cache_dtype: {video_pixel_cache_dtype}", + f"video_embed_path: {video_embed_path}", + f"source_video_fps: {source_video_fps}", + f"video_duration_sec: {video_duration_sec}", + f"full_video: {full_video}", + f"video_clip_motion_fps: {video_clip_motion_fps}", + "use_audio_in_video: False", + "output_root_quaternion: wxyz", + "output_joint_angles: 29D", + "output_root_position: xyz", + "", + f"frames: {frames}", + "", + ] + path.write_text("\n".join(lines), encoding="utf-8") + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description="Seed video->motion offline inference exporter") + parser.add_argument("--run-dir", type=str, required=True) + parser.add_argument("--ckpt-step", type=str, default="latest", help="integer step, 'latest', or 'best'") + parser.add_argument("--output-dir", type=str, required=True) + parser.add_argument("--motion-dir", type=str, required=True) + parser.add_argument("--video-path", type=str, default="") + parser.add_argument("--num-output-frames", type=int, default=0, help="0 means source motion length") + parser.add_argument("--num-inference-steps", type=int, default=10) + parser.add_argument("--device", type=str, default="cuda:0") + parser.add_argument("--seed", type=int, default=42) + parser.add_argument("--prompt", type=str, default="", help="Optional text instruction paired with the video") + parser.add_argument("--export-delta", action="store_true", default=False) + parser.add_argument("--disable-rtc", action="store_true", default=False) + parser.add_argument("--video-config-key", type=str, default="video", help="data config key: video or video_clip") + parser.add_argument("--video-clip", action="store_true", default=False, help="Use one aligned video clip per rollout chunk") + parser.add_argument( + "--video-pixel-cache-dtype", + choices=("auto", "none", "fp16", "bf16"), + default="auto", + help="Match cached Qwen pixel-value quantization; auto uses fp16 for cached video_clip runs.", + ) + parser.add_argument( + "--video-clip-motion-fps", + type=float, + default=0.0, + help="Override motion fps used to align rollout chunks to video clips; <=0 infers it from source motion/video.", + ) + return parser.parse_args() + + +def main() -> None: + args = parse_args() + + run_dir = Path(args.run_dir) + output_dir = Path(args.output_dir) + output_dir.mkdir(parents=True, exist_ok=True) + + cfg_template: LaunchConfig = parse_args_to_tyro_config(run_dir / "argv.txt") # type: ignore + launch_config = cfg_template.model_validate_json((run_dir / "run_config.json").read_text()) + seed_everything(launch_config.seed or args.seed) + + ckpt_step = resolve_ckpt_step(run_dir, args.ckpt_step) + model_cls = ( + MotionDistillPsi0Model + if hasattr(launch_config.model, "motion_distill_stage") + else Psi0Model + ) + model = model_cls.from_pretrained(run_dir, ckpt_step, launch_config, device=args.device) + model.text_only = False + model.audio_only = False + model.video_only = True + model.to(args.device) + model.eval() + + video_cfg = video_data_cfg(launch_config, args.video_config_key) + data_root = Path(cfg_get(video_cfg, "root_dir")) + video_root = Path(cfg_get(video_cfg, "video_dir", "")) + motion_dir = resolve_motion_dir(data_root, args.motion_dir) + video_path = resolve_video_path(video_root, motion_dir, args.video_path) + if not motion_dir.exists(): + raise FileNotFoundError(f"motion_dir does not exist: {motion_dir}") + if not video_path.exists(): + raise FileNotFoundError(f"video_path does not exist: {video_path}") + + maxmin = launch_config.data.transform.field + odim = int(launch_config.model.odim) + textop_actions = bool(getattr(launch_config.data.transform.repack, "textop_actions", False)) + decode_textop_joint_from_delta = textop_actions and should_decode_textop_joint_from_delta(launch_config) + num_past_frames = int(getattr(launch_config.data.transform.repack, "num_past_frames", 0)) + use_future_progress_embedding = bool(getattr(launch_config.model, "use_future_progress_embedding", False)) + enable_rtc = bool(getattr(launch_config.model, "rtc", False)) and (not args.disable_rtc) + action_exec_horizon = int(getattr(launch_config.model, "action_exec_horizon", int(model.action_horizon))) + rtc_max_delay = int(getattr(launch_config.model, "max_delay", 0)) + video_sample_fps = float(cfg_get(video_cfg, "video_sample_fps", 1.0)) + video_size_pixels = int(cfg_get(video_cfg, "video_size_pixels", 50176)) + video_decoder = str(cfg_get(video_cfg, "video_decoder", "opencv")) + if args.video_pixel_cache_dtype == "auto": + video_pixel_cache_dtype = ( + "fp16" + if args.video_clip and bool(cfg_get(video_cfg, "use_omni_hidden_cache", False)) + else "none" + ) + else: + video_pixel_cache_dtype = str(args.video_pixel_cache_dtype) + video_cfg_motion_fps = float(cfg_get(video_cfg, "motion_fps", TARGET_FPS)) + video_embed_dir = "" if args.video_clip else str(cfg_get(video_cfg, "video_embed_dir", "")) + instruction = args.prompt.strip() + + video_embed = None + video_embed_path = "" + if (not args.video_clip) and video_embed_dir.strip(): + cache_path = video_embed_cache_path( + video_path, + video_embed_dir, + video_sample_fps=video_sample_fps, + video_size_pixels=video_size_pixels, + ) + if cache_path.exists(): + video_embed = load_video_embed_cache(cache_path) + video_embed_path = str(cache_path) + + if args.video_clip: + _, _, source_video_fps, video_duration_sec = _probe_video(video_path) + video = None + elif video_embed is None: + video, source_video_fps, video_duration_sec = load_video_array( + video_path, + video_sample_fps=video_sample_fps, + video_size_pixels=video_size_pixels, + video_decoder=video_decoder, + ) + else: + video = None + source_video_fps = float(video_embed["source_video_fps"]) + video_duration_sec = float(video_embed["video_duration_sec"]) + + x0 = load_pose_from_motion_dir(motion_dir) + source_frames = load_source_frame_count(motion_dir) + target_frames = int(args.num_output_frames) if args.num_output_frames > 0 else int(source_frames or 0) + if args.video_clip and float(args.video_clip_motion_fps) > 0: + video_clip_motion_fps = float(args.video_clip_motion_fps) + elif args.video_clip and video_cfg_motion_fps > 0: + video_clip_motion_fps = float(video_cfg_motion_fps) + elif args.video_clip and source_frames > 0 and video_duration_sec > 0: + video_clip_motion_fps = float(source_frames) / float(video_duration_sec) + else: + video_clip_motion_fps = float(TARGET_FPS) + + pred_abs, pred_delta, rollout_chunks = rollout_video_motion( + model=model, + maxmin=maxmin, + odim=odim, + device=args.device, + video=video, + video_embed=video_embed, + video_path=video_path, + instruction=instruction, + video_size_pixels=video_size_pixels, + video_sample_fps=video_sample_fps, + video_decoder=video_decoder, + video_pixel_cache_dtype=video_pixel_cache_dtype, + video_clip_mode=bool(args.video_clip), + video_clip_motion_fps=float(video_clip_motion_fps), + normalize_state=bool(getattr(maxmin, "normalize_state", False)), + textop_actions=textop_actions, + decode_textop_joint_from_delta=decode_textop_joint_from_delta, + num_past_frames=num_past_frames, + use_future_progress_embedding=use_future_progress_embedding, + enable_rtc=enable_rtc, + action_exec_horizon=action_exec_horizon, + rtc_max_delay=rtc_max_delay, + num_inference_steps=int(args.num_inference_steps), + target_frames=target_frames, + init_abs_state=x0, + ) + + motion_name = sanitize_motion_name(motion_dir.name) + sample_dir = output_dir / motion_name + sample_dir.mkdir(parents=True, exist_ok=True) + + joint_pos = pred_abs[:, :29] + body_pos = pred_abs[:, 29:32] + body_quat = pred_abs[:, 32:36] + + dt = 1.0 / float(TARGET_FPS) + joint_vel = finite_difference(joint_pos, dt) + body_lin_vel = finite_difference(body_pos, dt) + body_ang_vel = quaternion_angular_velocity(body_quat, dt) + + write_csv(sample_dir / "joint_pos.csv", joint_pos, [f"joint_{i}" for i in range(29)]) + write_csv(sample_dir / "joint_vel.csv", joint_vel, [f"joint_vel_{i}" for i in range(29)]) + write_csv(sample_dir / "body_pos.csv", body_pos, ["body_0_x", "body_0_y", "body_0_z"]) + write_csv(sample_dir / "body_quat.csv", body_quat, ["body_0_w", "body_0_x", "body_0_y", "body_0_z"]) + write_csv(sample_dir / "body_lin_vel.csv", body_lin_vel, ["body_0_vel_x", "body_0_vel_y", "body_0_vel_z"]) + write_csv(sample_dir / "body_ang_vel.csv", body_ang_vel, ["body_0_angvel_x", "body_0_angvel_y", "body_0_angvel_z"]) + np.save(sample_dir / "motion.npy", pred_abs) + + if args.export_delta: + write_csv(sample_dir / "joint_pos_delta.csv", pred_delta[:, :29], [f"joint_{i}" for i in range(29)]) + write_csv(sample_dir / "body_pos_delta.csv", pred_delta[:, 29:32], ["body_0_x", "body_0_y", "body_0_z"]) + write_csv(sample_dir / "body_quat_delta.csv", pred_delta[:, 32:36], ["body_0_w", "body_0_x", "body_0_y", "body_0_z"]) + np.save(sample_dir / "motion_delta.npy", pred_delta) + + write_metadata(sample_dir / "metadata.txt", motion_name, int(pred_abs.shape[0])) + write_video_info( + sample_dir / "info.txt", + video_decoder=video_decoder, + source_motion_dir=motion_dir, + video_path=video_path, + instruction=instruction, + initial_pose_source=str(motion_dir), + frames=int(pred_abs.shape[0]), + textop_actions=textop_actions, + num_past_frames=int(num_past_frames), + enable_rtc=enable_rtc, + action_exec_horizon=int(action_exec_horizon), + num_inference_steps=int(args.num_inference_steps), + video_sample_fps=float(video_sample_fps), + video_size_pixels=int(video_size_pixels), + video_pixel_cache_dtype=video_pixel_cache_dtype, + video_embed_path=video_embed_path, + source_video_fps=float(source_video_fps), + video_duration_sec=float(video_duration_sec), + full_video=not bool(args.video_clip), + video_config_key=str(args.video_config_key), + video_clip_motion_fps=float(video_clip_motion_fps), + ) + + for src_name, dst_name in (("info.txt", "info_source.txt"), ("metadata.txt", "metadata_source.txt")): + src_path = motion_dir / src_name + if src_path.exists(): + shutil.copy2(src_path, sample_dir / dst_name) + + sample_meta = { + "run_dir": str(run_dir), + "ckpt_step": ckpt_step, + "source_motion_dir": str(motion_dir), + "video_path": str(video_path), + "instruction": instruction, + "output_dir": str(sample_dir), + "num_output_frames": int(pred_abs.shape[0]), + "rollout_chunks": int(rollout_chunks), + "textop_actions": textop_actions, + "decode_textop_joint_from_delta": decode_textop_joint_from_delta, + "num_past_frames": int(num_past_frames), + "enable_rtc": enable_rtc, + "action_exec_horizon": int(action_exec_horizon), + "video_sample_fps": float(video_sample_fps), + "video_size_pixels": int(video_size_pixels), + "video_pixel_cache_dtype": video_pixel_cache_dtype, + "video_decoder": video_decoder, + "video_embed_path": video_embed_path, + "source_video_fps": float(source_video_fps), + "video_duration_sec": float(video_duration_sec), + "full_video": not bool(args.video_clip), + "video_config_key": str(args.video_config_key), + "video_clip_motion_fps": float(video_clip_motion_fps), + "use_video_instruction": bool(cfg_get(video_cfg, "use_video_instruction", False)), + "use_audio_in_video": False, + } + (sample_dir / "infer_meta.json").write_text(json.dumps(sample_meta, ensure_ascii=False, indent=2), encoding="utf-8") + (output_dir / "manifest.json").write_text( + json.dumps({"samples": [sample_meta]}, ensure_ascii=False, indent=2), + encoding="utf-8", + ) + print(f"[OK] video sample -> {sample_dir}") + print(f"[DONE] wrote manifest: {output_dir / 'manifest.json'}") + + +if __name__ == "__main__": + main() diff --git a/scripts/infer/third_party_checkpoint_probe.py b/scripts/infer/third_party_checkpoint_probe.py new file mode 100644 index 0000000000000000000000000000000000000000..275840f59b9b305180181c241b35232ff95e9a33 --- /dev/null +++ b/scripts/infer/third_party_checkpoint_probe.py @@ -0,0 +1,434 @@ +#!/usr/bin/env python3 +"""Fixed train-only, closed-loop probes with durable prediction evidence. + +Called synchronously at a source boundary by the third-party trainer. No GT +pose after frame zero is fed to the model; no input waveform is re-mixed. +""" +from __future__ import annotations + +import argparse +import contextlib +import copy +import hashlib +import json +import os +from pathlib import Path +import random +import shutil +import subprocess +import sys +import time +import traceback +from typing import Any, Iterator, Sequence +import wave + +import numpy as np +import torch + +PROJECT_ROOT = Path(__file__).resolve().parents[2] +FPS = 50 +RATE = 16000 +PREVIEW_FPS = 25 +PREVIEW_WIDTH = 320 +PREVIEW_HEIGHT = 240 +PREVIEW_CPU_THREADS = 4 +PROFILES = ("single_short", "multi_single", "multi_babble", "mixed_short", + "continuous", "same_voice_aside") + + +def _sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as handle: + for chunk in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def _write_json(path: Path, value: Any) -> None: + temporary = path.with_suffix(path.suffix + ".tmp") + temporary.write_text(json.dumps(value, indent=2, ensure_ascii=False, + allow_nan=False) + "\n", encoding="utf-8") + os.replace(temporary, path) + + +def validate_probe_recipes(recipes: Sequence[dict[str, Any]], *, probe_scope: str = "checkpoint") -> None: + ids = [str(row["session_id"]) for row in recipes] + if probe_scope not in {"checkpoint", "smoke"}: + raise ValueError("Unknown probe scope") + if any(row.get("split") != "train" for row in recipes): + raise ValueError("Checkpoint probes may only use the selected training sessions") + if probe_scope == "smoke": + if not 1 <= len(ids) <= 12 or len(set(ids)) != len(ids): + raise ValueError("Smoke probes require 1–12 distinct train sessions") + if any(row.get("session_type") not in PROFILES for row in recipes): + raise ValueError("Unknown third-party smoke profile") + return + if len(ids) != 12 or len(set(ids)) != 12: + raise ValueError("Checkpoint probes require 12 distinct fixed train sessions") + for profile in PROFILES: + rows = [row for row in recipes if row.get("session_type") == profile] + if len(rows) != 2: + raise ValueError(f"Checkpoint probes require exactly two sessions for {profile}") + languages = {row["background_assignment"]["language"] for row in rows} + if languages != {"zh", "en"}: + raise ValueError(f"Checkpoint probes need zh/en backgrounds for {profile}") + + +def load_frozen_probe(production_root: Path, recipe: dict[str, Any]) -> tuple[ + np.ndarray, dict[str, np.ndarray], Path]: + """Hash-verify and decode PCM directly; never normalize or re-sample.""" + audio_path = (production_root / recipe["audio"]["path"]).resolve() + timeline_path = (production_root / recipe["timeline"]["path"]).resolve() + for path, expected in ((audio_path, recipe["audio"]["sha256"]), + (timeline_path, recipe["timeline"]["file_sha256"])): + if _sha256(path) != expected: + raise RuntimeError(f"Frozen probe input hash changed: {path}") + with wave.open(str(audio_path), "rb") as source: + if (source.getnchannels(), source.getsampwidth(), source.getframerate()) != (1, 2, RATE): + raise ValueError("Frozen probe audio must be mono 16 kHz PCM16") + if source.getnframes() != int(recipe["audio"]["sample_count"]): + raise ValueError("Frozen audio length no longer matches its recipe") + waveform = np.frombuffer(source.readframes(source.getnframes()), dtype=" dict[str, Any]: + """Report pose errors and control decisions separately, with explicit alignment.""" + predicted = np.asarray(motion, dtype=np.float64) + states = np.asarray(timeline["states"], dtype=np.float64) + labels = np.asarray(timeline["control_label"], dtype=np.int64) + if predicted.ndim != 2 or predicted.shape[1] != 36 or not np.isfinite(predicted).all(): + raise ValueError("Non-finite or malformed model prediction") + pose_count = min(len(predicted), len(states) - 1) + if pose_count <= 0: + raise ValueError("No aligned predicted/GT poses") + # Training actions at t predict t+1, while compact states store t. + delta = predicted[:pose_count] - states[1:pose_count + 1] + pred_quat = predicted[:pose_count, 32:36] + gt_quat = states[1:pose_count + 1, 32:36] + pred_norm = np.linalg.norm(pred_quat, axis=1) + gt_norm = np.linalg.norm(gt_quat, axis=1) + if np.any(pred_norm <= 1e-8) or np.any(gt_norm <= 1e-8): + raise ValueError("Zero root quaternion in predicted/GT pose") + quat_similarity = np.abs(np.sum(pred_quat * gt_quat, axis=1) / (pred_norm * gt_norm)) + count = min(len(predicted), len(labels)) + controls = np.full(len(predicted), -1, dtype=np.int64) + events = metadata["control_events"] + for index, event in enumerate(events): + start = int(event["output_frame"]) + end = int(events[index + 1]["output_frame"]) if index + 1 < len(events) else len(predicted) + control_id = int(event["predicted_id"]) + if not 0 <= start < end <= len(predicted) or control_id not in (0, 1, 2): + raise ValueError("Invalid prediction control timeline") + controls[start:end] = control_id + if np.any(controls < 0): + raise ValueError("Prediction control events do not cover the trajectory") + controls = controls[:count] + gt = labels[:count] + confusion = np.zeros((3, 3), dtype=np.int64) + np.add.at(confusion, (gt, controls), 1) + # Background-only is NOT synonymous with idle: a valid earlier command can + # still be executing. False activation is scored only when GT is non-motion. + sample_starts = np.arange(count) * (RATE // FPS) + background = np.zeros(count, dtype=bool) + target = np.zeros(count, dtype=bool) + for event in recipe.get("interference_events", []): + background |= ((sample_starts < int(event["end_sample"])) & + (sample_starts + RATE // FPS > int(event["start_sample"]))) + for event in recipe.get("target_speech_events", []): + target |= ((sample_starts < int(event["end_sample"])) & + (sample_starts + RATE // FPS > int(event["start_sample"]))) + background_nonmotion = background & ~target & (gt != 1) + no_motion = gt != 1 + motion_gt = gt == 1 + def rate(values: np.ndarray, mask: np.ndarray) -> float | None: + return float(values[mask].mean()) if mask.any() else None + return { + "pose_alignment": "predicted[t] compared with frozen state[t+1]; no DTW/realignment", + "control_alignment": "control predicted at t compared with frozen control_label[t]", + "pose_scored_frames": pose_count, "control_scored_frames": count, + "prediction_frames": len(predicted), + "unscored_prediction_tail_frames": len(predicted) - pose_count, + "joint_mae_radians": float(np.abs(delta[:, :29]).mean()), + "root_position_rmse_metres": float(np.sqrt(np.square(delta[:, 29:32]).sum(axis=1).mean())), + "root_rotation_mae_radians": float((2 * np.arccos(np.clip(quat_similarity, 0, 1))).mean()), + "control_accuracy": float((controls == gt).mean()), + "control_confusion_rows_gt_columns_prediction": confusion.tolist(), + "control_order": ["listen", "motion", "idle"], + "nonmotion_gt_frames": int(no_motion.sum()), + "false_motion_rate": rate(controls == 1, no_motion), + "motion_gt_frames": int(motion_gt.sum()), + "missed_motion_rate": rate(controls != 1, motion_gt), + "background_only_nonmotion_gt_frames": int(background_nonmotion.sum()), + "background_only_false_motion_rate": rate(controls == 1, background_nonmotion), + "root_height_min_metres": float(predicted[:, 31].min()), + "root_height_max_metres": float(predicted[:, 31].max()), + "joint_step_rms_radians": float(np.sqrt(np.square(np.diff(predicted[:, :29], axis=0)).mean())), + "interpretation": "Train memorization/robustness diagnostic, not held-out generalization or physics success.", + } + + +@contextlib.contextmanager +def preserve_training_state(model: Any, seed: int) -> Iterator[None]: + """Probe must not alter dropout flags, RNG streams, or training scheduler.""" + module_flags = [(module, module.training) for module in model.modules()] + python_rng, numpy_rng = random.getstate(), np.random.get_state() + torch_rng = torch.get_rng_state() + cuda_rng = torch.cuda.get_rng_state_all() if torch.cuda.is_initialized() else None + old_scheduler = getattr(model, "noise_scheduler", None) + try: + if old_scheduler is not None: + model.noise_scheduler = copy.deepcopy(old_scheduler) + random.seed(seed) + np.random.seed(seed % (2 ** 32)) + torch.manual_seed(seed) + model.eval() + with torch.inference_mode(): + yield + finally: + for module, training in module_flags: + module.training = training + if old_scheduler is not None: + model.noise_scheduler = old_scheduler + random.setstate(python_rng) + np.random.set_state(numpy_rng) + torch.set_rng_state(torch_rng) + if cuda_rng is not None: + torch.cuda.set_rng_state_all(cuda_rng) + + +@contextlib.contextmanager +def inference_bindings(model: Any, launch_config: Any, vlm_processor: Any) -> Iterator[None]: + """Training keeps processor/scheduler on the trainer, unlike loaded inference.""" + from diffusers.schedulers.scheduling_flow_match_euler_discrete import FlowMatchEulerDiscreteScheduler + + missing = object() + names = ("device", "vlm_processor", "noise_scheduler") + previous = {name: getattr(model, name, missing) for name in names} + cfg = getattr(launch_config, "model", None) + if str(getattr(cfg, "noise_scheduler", "flow")) != "flow": + raise ValueError("Closed-loop third-party probes require the flow scheduler") + processor = vlm_processor if vlm_processor is not None else getattr(model, "vlm_processor", None) + if processor is None: + raise ValueError("In-process checkpoint probe requires trainer.vlm_processor") + try: + model.device = next(model.parameters()).device + model.vlm_processor = processor + model.noise_scheduler = FlowMatchEulerDiscreteScheduler( + num_train_timesteps=int(getattr(cfg, "train_diffusion_steps", 1000))) + yield + finally: + for name, value in previous.items(): + if value is missing: + if hasattr(model, name): + delattr(model, name) + else: + setattr(model, name, value) + + +def _run(command: Sequence[str], log: Path, **kwargs: Any) -> None: + with log.open("a", encoding="utf-8") as handle: + result = subprocess.run(list(command), stdout=handle, stderr=subprocess.STDOUT, + check=False, timeout=1800, **kwargs) + if result.returncode: + raise RuntimeError(f"Probe render failed (exit {result.returncode}); see {log}") + + +def render_prediction(sample_dir: Path, audio_path: Path) -> Path: + """Low-cost CPU review video; the underlying predicted CSV remains 50 Hz.""" + from scripts.data.render_hfd_third_party_gt import ( + GMR_PYTHON, GMR_XML, RENDER_ROOT, _render_environment, + ) + from scripts.infer.mux_full_duplex_audio import probe as probe_media, stream + from scripts.third_party_temp import project_temp_directory + + environment = _render_environment("cpu", cpu_threads=PREVIEW_CPU_THREADS) + scratch = str(project_temp_directory(PROJECT_ROOT)) + environment.update({name: scratch for name in ("TMPDIR", "TMP", "TEMP", "SONIC_RENDER_TMPDIR")}) + meta_path = sample_dir / "infer_meta.json" + metadata = json.loads(meta_path.read_text(encoding="utf-8")) + metadata["review_render"] = { + "motion_fps": FPS, "video_fps": PREVIEW_FPS, + "width": PREVIEW_WIDTH, "height": PREVIEW_HEIGHT, + "cpu_threads": PREVIEW_CPU_THREADS, + "purpose": "Checkpoint screening preview; full 50Hz predicted poses and metrics retained", + "audio_policy": "Complete original mixed WAV, no time stretch or offset", + } + _write_json(meta_path, metadata) + log = sample_dir / "render.log" + _run([str(GMR_PYTHON), str(RENDER_ROOT / "scripts/batch_render_reference_motion_mujoco.py"), + "--motion-dir", str(sample_dir), "--xml-path", str(GMR_XML), + "--source-fps", str(FPS), "--target-fps", str(PREVIEW_FPS), + "--width", str(PREVIEW_WIDTH), "--height", str(PREVIEW_HEIGHT), + "--dof-order", "isaaclab", "--camera-distance", "2.5"], log, + cwd=sample_dir, env=environment) + # The shared renderer calls every CSV input 'reference'; rename it here so + # users do not confuse the actual model prediction with the dataset's GT. + silent = sample_dir / "prediction_silent.mp4" + (sample_dir / "reference_motion.mp4").replace(silent) + media = probe_media(silent) + video_stream = stream(media, "video") + with wave.open(str(audio_path), "rb") as source: + audio_seconds = source.getnframes() / source.getframerate() + video_seconds = float(video_stream.get("duration") or media["format"]["duration"]) + if (video_stream["width"], video_stream["height"]) != (PREVIEW_WIDTH, PREVIEW_HEIGHT): + raise RuntimeError("Checkpoint preview resolution drifted") + if video_seconds < audio_seconds - 1e-6 or video_seconds - audio_seconds > 1 / PREVIEW_FPS + 1e-6: + raise RuntimeError("Checkpoint preview would truncate or extend the original audio timeline") + video = sample_dir / "prediction_with_audio.mp4" + _run([sys.executable, str(PROJECT_ROOT / "scripts/infer/mux_full_duplex_audio.py"), + "--video", str(silent), "--audio", str(audio_path), "--output", str(video), + "--infer-meta", str(sample_dir / "infer_meta.json"), "--audio-is-timeline", + "--expected-fps", str(PREVIEW_FPS)], log, env=environment) + _run(["ffmpeg", "-v", "error", "-xerror", "-i", str(video), + "-f", "null", "-"], log, env=environment) + return video + + +def _aggregate(rows: Sequence[dict[str, Any]]) -> dict[str, Any]: + keys = ("joint_mae_radians", "root_position_rmse_metres", "control_accuracy", + "root_rotation_mae_radians", + "false_motion_rate", "missed_motion_rate", "background_only_false_motion_rate") + result: dict[str, Any] = {"averaging": "unweighted mean over available session metrics"} + for key in keys: + values = [row["metrics"][key] for row in rows if row["metrics"].get(key) is not None] + result[key] = float(np.mean(values)) if values else None + result[key + "_sessions"] = len(values) + return result + + +def run_checkpoint_probes(model: Any, launch_config: Any, *, production_root: str | Path, + recipes: Sequence[dict[str, Any]], output_root: str | Path, + global_step: int, seed: int = 260930, + num_inference_steps: int = 10, + probe_scope: str = "checkpoint", + vlm_processor: Any = None) -> dict[str, Any]: + """Run all fixed 12 predictions; persist partial failure and re-raise errors. + + The trainer must call only at a source boundary, and pass recipes from its + frozen 300-session selection. output_root is normally run/checkpoint_probes. + """ + from scripts.infer import full_duplex_audio_to_motion as infer + + validate_probe_recipes(recipes, probe_scope=probe_scope) + if getattr(model, "control_condition_inference_mode", "hard") != "hard": + raise ValueError("Third-party checkpoint probes require own hard control conditioning") + if int(global_step) < 0 or int(num_inference_steps) <= 0: + raise ValueError("Invalid probe step/inference steps") + production_root = Path(production_root).resolve() + output_root = Path(output_root).resolve() + if not output_root.is_relative_to(PROJECT_ROOT.resolve()): + raise ValueError("Probe outputs must remain inside HFD_lzk") + step_dir = output_root / f"step_{int(global_step):08d}" + step_dir.mkdir(parents=True, exist_ok=False) + started = time.monotonic() + report: dict[str, Any] = { + "format": "hfd_third_party_checkpoint_probe_v1", "status": "running", + "global_step": int(global_step), "seed": int(seed), "evaluation_split": "train_probe", + "probe_scope": probe_scope, + "fixed_session_ids": [row["session_id"] for row in recipes], + "input_policy": "Exact frozen mixed PCM16; no remix, resampling, gain change, or video input", + "rollout_policy": "Own hard control and predicted state/RTC; only initial GT pose supplied", + "tail_policy": "Final 0.2s uses last predicted chunk tail, not a GT or stationary hold", + "render_backend": "mesa_llvmpipe_cpu", "motion_fps": FPS, + "review_video_fps": PREVIEW_FPS, "review_video_resolution": [PREVIEW_WIDTH, PREVIEW_HEIGHT], + "render_cpu_threads": PREVIEW_CPU_THREADS, "samples": [], + } + _write_json(step_dir / "report.json", report) + try: + with preserve_training_state(model, seed), inference_bindings(model, launch_config, vlm_processor): + for index, recipe in enumerate(recipes, 1): + session_id = str(recipe["session_id"]) + sample_dir = step_dir / f"{index:02d}_{recipe['session_type']}_{session_id[:12]}" + sample_dir.mkdir() + waveform, timeline, audio_path = load_frozen_probe(production_root, recipe) + print(f"[train_probe] step={global_step} sample={index}/{len(recipes)} session={session_id}", flush=True) + motion, metadata = infer.rollout_full_duplex( + model=model, background_waveforms=[], waveform=waveform, + prompt=infer.DEFAULT_COMMAND_PROMPT, source_id=f"train_probe:{session_id}", + init_abs=timeline["states"][0].copy(), launch_config=launch_config, + motion_frames=max(int(getattr(launch_config.data, "turn_transition_frames", 50)), 50) + 300, + num_inference_steps=int(num_inference_steps), respect_control_token=True, + max_listen_tail_steps=1, initial_idle_steps=0, + seed=seed + index, frozen_audio_timeline=True, + ) + expected = int(round(len(waveform) * FPS / RATE)) + if motion.shape != (expected, 36): + raise RuntimeError("Frozen probe prediction does not cover its exact audio timeline") + metrics = score_prediction(motion, metadata, timeline, recipe) + metadata.update({"session_id": session_id, "global_step": int(global_step), + "frozen_audio_path": str(audio_path), + "frozen_audio_sha256": recipe["audio"]["sha256"], + "artifact_role": "model_prediction_not_GT"}) + infer.export_motion(sample_dir, motion) + _write_json(sample_dir / "infer_meta.json", metadata) + _write_json(sample_dir / "metrics.json", metrics) + _write_json(sample_dir / "session_recipe.json", recipe) + video = render_prediction(sample_dir, audio_path) + row = {"session_id": session_id, "profile": recipe["session_type"], + "background_language": recipe["background_assignment"]["language"], + "background_sir_db": recipe.get("mix_metadata", {}).get("measured_background_sir_db"), + "metrics": metrics, "video": str(video), "video_sha256": _sha256(video), + "motion_sha256": _sha256(sample_dir / "motion.npy"), + "full_video_decode_passed": True} + report["samples"].append(row) + _write_json(step_dir / "report.json", report) + report["status"] = "complete" + report["aggregate"] = _aggregate(report["samples"]) + except BaseException as error: + report["status"] = "failed" + report["error"] = f"{type(error).__name__}: {error}" + (step_dir / "failure.txt").write_text(traceback.format_exc(), encoding="utf-8") + raise + finally: + report["elapsed_seconds"] = time.monotonic() - started + _write_json(step_dir / "report.json", report) + _write_json(output_root / "latest.json", {"report": str(step_dir / "report.json"), + "status": report["status"], "global_step": global_step}) + with (output_root / "history.jsonl").open("a", encoding="utf-8") as handle: + handle.write(json.dumps({"global_step": global_step, "status": report["status"], + "report": str(step_dir / "report.json"), + "elapsed_seconds": report["elapsed_seconds"], + "aggregate": report.get("aggregate", {})}, ensure_ascii=False) + "\n") + lines = ["# Checkpoint 训练集闭环记录", "", f"step={global_step},状态:{report['status']}。", + "这些是模型预测,不是 GT;训练集结果不代表泛化能力。", "", + "审核视频为 320×240 / 25 fps;完整预测轨迹和指标保留 50 Hz,原始混音时间线不变。", "", + "| 序号 | 类型 | 背景语言 | 控制准确率 | 视频 |", "|---|---|---|---:|---|"] + for index, row in enumerate(report["samples"], 1): + relative_video = Path(row["video"]).relative_to(step_dir) + lines.append(f"| {index:02d} | {row['profile']} | {row['background_language']} | " + f"{row['metrics']['control_accuracy']:.3f} | [模型预测]({relative_video}) |") + if report.get("error"): + lines.extend(["", "失败原因见 failure.txt;未生成的样本不得视为通过。"]) + (step_dir / "README.md").write_text("\n".join(lines) + "\n", encoding="utf-8") + return report + + +def check_renderer() -> dict[str, Any]: + from scripts.data.render_hfd_third_party_gt import GMR_PYTHON, GMR_XML, RENDER_ROOT + required = (GMR_PYTHON, GMR_XML, RENDER_ROOT / "scripts/batch_render_reference_motion_mujoco.py") + missing = [str(path) for path in required if not path.is_file()] + missing += [name for name in ("ffmpeg", "ffprobe") if not shutil.which(name)] + if missing: + raise FileNotFoundError(f"Missing checkpoint video renderer requirements: {missing}") + return {"paths_present": True, "backend": "cpu", "render_execution_verified": False} + + +if __name__ == "__main__": + if str(PROJECT_ROOT) not in sys.path: + sys.path.insert(0, str(PROJECT_ROOT)) + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--check-renderer", action="store_true", required=True) + parser.parse_args() + print(json.dumps(check_renderer(), ensure_ascii=False)) diff --git a/scripts/infer/third_party_scale_metrics.py b/scripts/infer/third_party_scale_metrics.py new file mode 100644 index 0000000000000000000000000000000000000000..8b7b3af95fafe34670309af878c04e23ecfe1043 --- /dev/null +++ b/scripts/infer/third_party_scale_metrics.py @@ -0,0 +1,172 @@ +#!/usr/bin/env python3 +"""GPU-0 scale-stage closed-loop metrics on separately frozen train/validation.""" +from __future__ import annotations + +import argparse +import hashlib +import json +import os +from pathlib import Path +import subprocess +import sys +import time + +PROJECT = Path(__file__).resolve().parents[2] +sys.path[:0] = [str(PROJECT), str(PROJECT / "src")] +from scripts.third_party_strategy_plan import write_json +from scripts.infer.third_party_metrics import aggregate, local +from scripts.third_party_scale_plan import SEED, safe_relative_path + + +def read(path): + return json.loads(Path(path).read_text()) + + +def sample_seed(sid): + return int(hashlib.sha256(f"{SEED}:rollout:{sid}".encode()).hexdigest()[:8], 16) + + +def recipes_for(manifest, ids): + from psi.data.seed.third_party_frozen_dataset import read_jsonl, sha256_file + selection = read(manifest) + split = selection.get("split") + if (selection.get("scaleup") is not True or split not in ("train", "validation") + or len(set(ids)) != len(ids) or not set(ids) <= set(selection["session_ids"])): + raise ValueError("Invalid scale evaluation split/IDs") + root = local(selection["production_root"]) + path = root / "manifests/session_recipes.jsonl" + if sha256_file(path) != selection["recipe_manifest_sha256"]: + raise ValueError("Evaluation recipe manifest changed") + wanted, found = set(ids), {} + for row in read_jsonl(path): + if row["session_id"] in wanted: + if row["session_id"] in found or row["split"] != split: + raise ValueError("Evaluation crossed production split") + for name in ("audio", "timeline"): + safe_relative_path(root, row[name]["path"]) + found[row["session_id"]] = row + if set(found) != wanted: + raise ValueError("Missing evaluation recipe") + return root, found, split + + +def evaluate(source_run, step, manifest, output, *, full=False, workers=4, include_videos=False): + if os.environ.get("CUDA_VISIBLE_DEVICES") != "0" or workers not in (1, 2, 4): + raise RuntimeError("Use the managed single-GPU-0 scale launcher") + source_run, manifest, output = map(local, (source_run, manifest, output)) + if output.exists() or not output.is_relative_to(PROJECT / "runs"): + raise ValueError("Fresh local evaluation directory required") + selection = read(manifest) + ids = list(selection["session_ids"] if full else selection["evaluation_session_ids"]) + if include_videos: + ids.extend(sid for sid in selection["video_session_ids"] if sid not in ids) + if not ids or len(set(ids)) != len(ids): + raise ValueError("Invalid fixed evaluation panel") + output.mkdir(parents=True) + (output / "samples").mkdir() + task = {"source_run": str(source_run), "step": step, "manifest": str(manifest), + "manifest_sha256": hashlib.sha256(manifest.read_bytes()).hexdigest(), + "output": str(output), "split": selection["split"], "all_ids": ids} + write_json(output / "identity.json", task) + started = time.monotonic() + children, logs = [], [] + try: + for rank in range(min(workers, len(ids))): + path = output / f"worker_{rank}.json" + write_json(path, {**task, "ids": ids[rank::workers]}) + log = (output / f"worker_{rank}.log").open("x"); logs.append(log) + children.append(subprocess.Popen([sys.executable, str(Path(__file__).resolve()), "--worker", str(path)], + cwd=PROJECT, stdout=log, stderr=subprocess.STDOUT)) + while any(child.poll() is None for child in children): + if any(child.poll() not in (None, 0) for child in children): + raise RuntimeError(f"Scale evaluation worker failed: {output}") + time.sleep(1) + if any(child.returncode != 0 for child in children): + raise RuntimeError(f"Scale evaluation failed: {output}") + finally: + for child in children: + if child.poll() is None: + child.terminate() + for child in children: + try: + child.wait(timeout=30) + except subprocess.TimeoutExpired: + child.kill(); child.wait() + for log in logs: + log.close() + rows = [read(output / "samples" / sid / "result.json") for sid in ids] + by_id = {r["session_id"]: r for r in selection["sessions"]} + groups = {} + for field in ("session_type", "background_language", "background_high"): + groups[field] = {value: {"count": len(subset), **aggregate(subset)} + for value in sorted({str(by_id[r["session_id"]][field]) for r in rows}) + if (subset := [r for r in rows if str(by_id[r["session_id"]][field]) == value])} + report = {"status": "complete", "step": step, "evaluation_split": selection["split"], + "panel": "all_eligible" if full else "fixed_class_language_panel", + "count": len(rows), "samples": rows, "aggregate": aggregate(rows), "groups": groups, + "elapsed_seconds": time.monotonic() - started, "workers": workers, "rendered_videos": 0, + "input_policy": "original full mixed WAV; no GT after initial pose; no optimizer updates", + "seed_policy": "stable per session, independent of workers/checkpoint/panel size"} + panel = set(selection["evaluation_session_ids"]) + fixed = [r for r in rows if r["session_id"] in panel] + report["fixed_panel_comparison"] = {"count": len(fixed), "aggregate": aggregate(fixed)} + write_json(output / "report.json", report) + return report + + +def worker(path): + import numpy as np + import torch + from scripts.third_party_temp import configure_project_temp + from scripts.infer import full_duplex_audio_to_motion as infer + from scripts.infer.third_party_checkpoint_probe import load_frozen_probe, score_prediction, _sha256 + from psi.models.full_duplex_psi0 import FullDuplexPsi0Model + from psi.config.train.posttrain_third_party_config import DynamicLaunchConfig + if os.environ.get("CUDA_VISIBLE_DEVICES") != "0": + raise RuntimeError("Scale metrics require GPU0 isolation") + configure_project_temp(PROJECT); torch.set_num_threads(2) + task = read(local(path)); manifest = local(task["manifest"]) + if hashlib.sha256(manifest.read_bytes()).hexdigest() != task["manifest_sha256"]: + raise ValueError("Metric task selection changed") + source = local(task["source_run"]) + cfg = DynamicLaunchConfig.model_validate_json((source / "run_config.json").read_text()) + infer.validate_contract(cfg) + model = FullDuplexPsi0Model.from_streaming_checkpoint(source, task["step"], cfg, torch.device("cuda:0")) + model.eval() + root, recipes, split = recipes_for(manifest, task["ids"]) + if split != task["split"]: + raise ValueError("Metric task split changed") + with torch.inference_mode(): + for sid in task["ids"]: + started = time.monotonic(); recipe = recipes[sid] + dest = local(task["output"]) / "samples" / sid; dest.mkdir() + waveform, timeline, audio = load_frozen_probe(root, recipe) + seed = sample_seed(sid) + motion, metadata = infer.rollout_full_duplex(model=model, background_waveforms=[], waveform=waveform, + prompt=infer.DEFAULT_COMMAND_PROMPT, source_id=f"scale_{split}:{sid}", + init_abs=timeline["states"][0].copy(), launch_config=cfg, + motion_frames=max(int(cfg.data.turn_transition_frames), 50) + 300, + num_inference_steps=10, respect_control_token=True, max_listen_tail_steps=1, + initial_idle_steps=0, seed=seed, frozen_audio_timeline=True) + if motion.shape != (round(len(waveform) * 50 / 16000), 36) or not np.isfinite(motion).all(): + raise ValueError("Incomplete/nonfinite scale prediction") + metrics = score_prediction(motion, metadata, timeline, recipe) + metrics["interpretation"] = ("Held-out production validation; not physics success" if split == "validation" + else "Training fit diagnostic, not held-out generalization") + metadata.update(session_id=sid, global_step=task["step"], evaluation_split=split, + frozen_audio_path=str(audio), frozen_audio_sha256=recipe["audio"]["sha256"], + artifact_role="model_prediction_not_GT") + infer.export_motion(dest, motion) + for name, value in (("infer_meta", metadata), ("metrics", metrics), ("session_recipe", recipe)): + write_json(dest / f"{name}.json", value) + row = {"session_id": sid, "profile": recipe["session_type"], "split": split, + "background_language": recipe["background_assignment"]["language"], "seed": seed, + "metrics": metrics, "prediction_directory": str(dest), "motion_sha256": _sha256(dest / "motion.npy"), + "render_status": "not_requested", "inference_seconds": time.monotonic() - started} + write_json(dest / "result.json", row) + print(json.dumps({"session": sid, "split": split, "seconds": row["inference_seconds"]}), flush=True) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(); parser.add_argument("--worker", required=True) + worker(parser.parse_args().worker) diff --git a/src/InternVLA-M1/InternVLA/__init__.py b/src/InternVLA-M1/InternVLA/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/src/InternVLA-M1/InternVLA/config/deepseeds/deepspeed_zero2.yaml b/src/InternVLA-M1/InternVLA/config/deepseeds/deepspeed_zero2.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6b4fb8e89063d69846d37e394e5087638137438b --- /dev/null +++ b/src/InternVLA-M1/InternVLA/config/deepseeds/deepspeed_zero2.yaml @@ -0,0 +1,10 @@ +compute_environment: LOCAL_MACHINE +debug: false +deepspeed_config: + deepspeed_config_file: "InternVLA/config/deepseeds/ds_config.yaml" + deepspeed_multinode_launcher: standard + zero3_init_flag: false +distributed_type: DEEPSPEED +# main_process_port: 12345 +num_machines: 1 +num_processes: 8 \ No newline at end of file diff --git a/src/InternVLA-M1/InternVLA/config/deepseeds/ds_config.yaml b/src/InternVLA-M1/InternVLA/config/deepseeds/ds_config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..95ac43cc81d8f8695c0f9d4d756bb8188e3995ed --- /dev/null +++ b/src/InternVLA-M1/InternVLA/config/deepseeds/ds_config.yaml @@ -0,0 +1,28 @@ +{ + "fp16": { + "enabled": "auto", + "loss_scale": 0, + "loss_scale_window": 1000, + "initial_scale_power": 16, + "hysteresis": 2, + "min_loss_scale": 1 + }, + "bf16": { + "enabled": "auto" + }, + "train_micro_batch_size_per_gpu": "auto", + "train_batch_size": "auto", + "gradient_accumulation_steps": 1, + "zero_optimization": { + "stage": 2, + "allgather_partitions": true, + "allgather_bucket_size": 5e8, + "reduce_scatter": true, + "reduce_bucket_size": 5e8, + "overlap_comm": true, + "contiguous_gradients": true, + "cpu_offload": false + }, + "gradient_clipping": 1.0, + "steps_per_print": 10 +} \ No newline at end of file diff --git a/src/InternVLA-M1/InternVLA/config/deepseeds/zero2.yaml b/src/InternVLA-M1/InternVLA/config/deepseeds/zero2.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f4df9d409dfc682961b2e9e970721a75cdd3667f --- /dev/null +++ b/src/InternVLA-M1/InternVLA/config/deepseeds/zero2.yaml @@ -0,0 +1,21 @@ +compute_environment: LOCAL_MACHINE +debug: false +deepspeed_config: + deepspeed_multinode_launcher: standard + offload_optimizer_device: none + offload_param_device: none + zero3_init_flag: false + zero_stage: 2 +distributed_type: DEEPSPEED +downcast_bf16: 'no' +machine_rank: 0 +main_training_function: main +mixed_precision: bf16 +num_machines: 1 +num_processes: 8 +rdzv_backend: static +same_network: true +tpu_env: [] +tpu_use_cluster: false +tpu_use_sudo: false +use_cpu: false \ No newline at end of file diff --git a/src/InternVLA-M1/InternVLA/config/deepseeds/zero3.json b/src/InternVLA-M1/InternVLA/config/deepseeds/zero3.json new file mode 100644 index 0000000000000000000000000000000000000000..6917317af62da757ca759a92b326ddfa65b203cc --- /dev/null +++ b/src/InternVLA-M1/InternVLA/config/deepseeds/zero3.json @@ -0,0 +1,28 @@ +{ + "fp16": { + "enabled": "auto", + "loss_scale": 0, + "loss_scale_window": 1000, + "initial_scale_power": 16, + "hysteresis": 2, + "min_loss_scale": 1 + }, + "bf16": { + "enabled": "auto" + }, + "train_micro_batch_size_per_gpu": "auto", + "train_batch_size": "auto", + "gradient_accumulation_steps": "auto", + "zero_optimization": { + "stage": 3, + "overlap_comm": true, + "contiguous_gradients": true, + "sub_group_size": 1e9, + "reduce_bucket_size": "auto", + "stage3_prefetch_bucket_size": "auto", + "stage3_param_persistence_threshold": "auto", + "stage3_max_live_parameters": 1e9, + "stage3_max_reuse_distance": 1e9, + "stage3_gather_16bit_weights_on_model_save": true + } +} \ No newline at end of file diff --git a/src/InternVLA-M1/InternVLA/config/training/internvla_cotrain_custom.yaml b/src/InternVLA-M1/InternVLA/config/training/internvla_cotrain_custom.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9e57c6c800656a5389d86981062d24e993d793fe --- /dev/null +++ b/src/InternVLA-M1/InternVLA/config/training/internvla_cotrain_custom.yaml @@ -0,0 +1,126 @@ +run_id: internvla_cotrain_custom +run_root_dir: results/Checkpoints +seed: 42 +trackers: [jsonl, wandb] +wandb_entity: WANDB_ENTITY +wandb_project: WANDB_PROJECT +is_debug: false + +framework: + framework_py: InternVLA-M1 + qwenvl: # TODO later should not use qwenvl should use vlm directly + base_vlm: ./playground/Pretrained_models/Qwen2.5-VL-3B-Instruct + attn_implementation: flash_attention_2 + vl_hidden_dim: 2048 # TODO should read from internal config + dino: + dino_backbone: dinov2_vits14 + layer_qformer: + qformer_end_layer: 37 + qformer_start_layer: 36 + num_query_tokens: 64 + input_dim: 2048 # here is VLM output dimension + ouptput_dim: 768 # here is action input dimension # here has a rule that ouptput_dim = action_hidden_dim; --> top2down logic should not default match, should be modified manually + grad_scale: 0.5 # let gradient pass through this module to decay, avoid destroying VLM, # seems to affect learning + + action_model: + action_model_type: DiT-B + action_hidden_dim: 768 + action_dim: 7 + use_ema: false + future_action_window_size: 15 + past_action_window_size: 0 + repeated_diffusion_steps: 8 + + fm_head_config: + input_embedding_dim: 1536 # Action embedding after dimension 48 * 32,and DiT's attention_head_dim * attention_heads corresponds to ActionEncoder + hidden_size: 1024 # and DiT's last projection corresponds to ActionDecoder + add_pos_embed: True + max_seq_len: 1024 + action_dim: 7 + future_action_window_size: 15 + action_horizon: 16 # = future_action_window_size + 1 + past_action_window_size + past_action_window_size: 0 + noise_beta_alpha: 1.5 + noise_beta_beta: 1.0 + noise_s: 0.999 + num_timestep_buckets: 1000 + num_inference_timesteps: 4 + num_target_vision_tokens: 32 + diffusion_model_cfg: # DiT Transformers parameters + attention_head_dim: 48 + cross_attention_dim: 2048 # VLM dim + dropout: 0.2 + final_dropout: true + interleave_self_attention: true + norm_type: "ada_norm" + num_attention_heads: 32 + num_layers: 16 + output_dim: 1024 + positional_embeddings: null + +datasets: + vlm_data: + dataset_py: vlm_datasets + dataformat: llava_json + dataset_use: asv2_conversation_en,asv2_detailed_description_en,asv2_region_captioning_en,coco_internvl_longcap_en,coco_karpathy_train_567_en,coco_negative_gpt4o_en,coco_poetry_zh,coco_rem_en_zh,cocorem_exist_yorn_en,cocotextv2_en,cocotextv2_gpt4o_en,okvqa_en,refcoco_grounding_aug_en,refcoco_grounding_en,tallyqa_coco_en,toloka_grounding_aug_en,vqav2_en,vsr_en + eval_dataset: aokvqa_cauldron_llava_format + data_flatten: false + base_interval: 2 + max_pixels: 50176 + min_pixels: 784 + model_max_length: 2048 # seems to be 4096 here, some related content is not processed + model_type: qwen2.5vl + per_device_batch_size: 4 + + vla_data: + dataset_py: lerobot_datasets + data_root_dir: playground/Datasets/YOUR_DATASET_PATH + data_mix: YOUR_DATASET_NAME + action_type: delta_ee + CoT_prompt: "Your task is {instruction}. To identify the key objects for your task. Locate their bounding boxes in [x1,y1,x2,y2] format." + CoT_answer: bbox + default_image_resolution: [3, 224, 224] # this is not effective + per_device_batch_size: 16 + load_all_data_for_training: true # consider how to enable eval set + obs: ["image_0"] # this is not effective + image_size: [224,224] #todo is null keep raw size # this is not effective + +trainer: + epochs: 100 # decide whether to use, he and max_train_steps conflict + max_train_steps: 100000 + num_warmup_steps: 5000 + save_interval: 5000 + eval_interval: 100 + learning_rate: # you can set different lr for different modules + base: 5e-05 + qwen_vl_interface: 1.0e-05 + action_model: 1.0e-04 + lr_scheduler_type: cosine_with_min_lr + scheduler_specific_kwargs: + min_lr: 5.0e-07 + freeze_modules: '' + loss_scale: + vla: 1.0 + vlm: 0.1 + # pretrained_checkpoint: ./results/Checkpoints/1_need/0601_qwenact_fixqwen_32gpus_lr_1e-3_qformer_36_37/steps_100000/pytorch_model.pt + # reload_modules: 'action_model,layer_qformer' # warning: this may cause model to not converge + repeated_diffusion_steps: 4 + max_grad_norm: 1.0 + warmup_ratio: 0.1 + weight_decay: 0.0 + logging_frequency: 10 + gradient_clipping: 1.0 + gradient_accumulation_steps: 1 + + optimizer: + name: AdamW + betas: [0.9, 0.95] + eps: 1.0e-08 + weight_decay: 1.0e-08 + + # parameters to be determined + is_resume: false + resume_epoch: null + resume_step: null + enable_gradient_checkpointing: true + enable_mixed_precision_training: true diff --git a/src/InternVLA-M1/InternVLA/config/training/internvla_cotrain_humanoid.yaml b/src/InternVLA-M1/InternVLA/config/training/internvla_cotrain_humanoid.yaml new file mode 100644 index 0000000000000000000000000000000000000000..25ea9c9fecefb4d8aae5c18f656d48ce02a17432 --- /dev/null +++ b/src/InternVLA-M1/InternVLA/config/training/internvla_cotrain_humanoid.yaml @@ -0,0 +1,128 @@ +run_id: humanoid_g1 +run_root_dir: runs/InternVLA/Checkpoints +seed: 42 +trackers: [jsonl, wandb] +wandb_entity: None +wandb_project: None +is_debug: false + +framework: + framework_py: InternVLA-M1 + qwenvl: # TODO later should not use qwenvl should use vlm directly + base_vlm: Qwen/Qwen2.5-VL-3B-Instruct + attn_implementation: flash_attention_2 + vl_hidden_dim: 2048 # TODO should read from internal config + dino: + dino_backbone: dinov2_vits14 + layer_qformer: + qformer_end_layer: 37 + qformer_start_layer: 36 + num_query_tokens: 64 + input_dim: 2048 # here is VLM output dimension + ouptput_dim: 768 # here is action input dimension # here has a rule that ouptput_dim = action_hidden_dim; --> top2down logic should not default match, should be modified manually + grad_scale: 0.5 # let gradient pass through this module to decay, avoid destroying VLM, # seems to affect learning + + action_model: + action_model_type: DiT-B + action_hidden_dim: 768 + action_dim: 36 # 32 + use_ema: false + future_action_window_size: 15 # 29 + past_action_window_size: 0 + repeated_diffusion_steps: 8 + + fm_head_config: + input_embedding_dim: 1536 # Action embedding after dimension 48 * 32,and DiT's attention_head_dim * attention_heads corresponds to ActionEncoder + hidden_size: 1024 # and DiT's last projection corresponds to ActionDecoder + add_pos_embed: True + max_seq_len: 1024 + action_dim: 36 # 32 + future_action_window_size: 15 # 29 + action_horizon: 16 # 30 # = future_action_window_size + 1 + past_action_window_size + past_action_window_size: 0 + noise_beta_alpha: 1.5 + noise_beta_beta: 1.0 + noise_s: 0.999 + num_timestep_buckets: 1000 + num_inference_timesteps: 4 + num_target_vision_tokens: 32 + diffusion_model_cfg: # DiT Transformers parameters + attention_head_dim: 48 + cross_attention_dim: 2048 # VLM dim + dropout: 0.2 + final_dropout: true + interleave_self_attention: true + norm_type: "ada_norm" + num_attention_heads: 32 + num_layers: 16 + output_dim: 1024 + positional_embeddings: null + +datasets: + vlm_data: + dataset_py: vlm_datasets + dataformat: llava_json + dataset_use: asv2_conversation_en,asv2_detailed_description_en,asv2_region_captioning_en,coco_internvl_longcap_en,coco_karpathy_train_567_en,coco_negative_gpt4o_en,coco_poetry_zh,coco_rem_en_zh,cocorem_exist_yorn_en,cocotextv2_en,cocotextv2_gpt4o_en,okvqa_en,refcoco_grounding_aug_en,refcoco_grounding_en,tallyqa_coco_en,toloka_grounding_aug_en,vqav2_en,vsr_en + eval_dataset: aokvqa_cauldron_llava_format + data_flatten: false + base_interval: 2 + max_pixels: 50176 + min_pixels: 784 + model_max_length: 2048 # seems to be 4096 here, some related content is not processed + model_type: qwen2.5vl + per_device_batch_size: 4 + + vla_data: + dataset_py: lerobot_datasets + data_root_dir: None + data_mix: humanoid_ + action_type: abs_joints + CoT_prompt: "Your task is {instruction}. To identify the key objects for your task. Locate their bounding boxes in [x1,y1,x2,y2] format." + CoT_answer: bbox + default_image_resolution: [3, 224, 224] # this is not effective + per_device_batch_size: 64 # FIXME + preload_all: true #FIXME + load_all_data_for_training: true # consider how to enable eval set + obs: ["image_0"] # this is not effective + image_size: [224,224] #todo is null keep raw size # this is not effective + +trainer: + epochs: 100 # not effective; use max_train_steps instead + max_train_steps: 30000 #100000 #FIXME + num_warmup_steps: 0 + save_interval: 5000 # 500 + eval_interval: 100 + learning_rate: # you can set different lr for different modules #FIXME + base: 5e-05 + qwen_vl_interface: 1.0e-05 + action_model: 1.0e-04 + lr_scheduler_type: cosine_with_min_lr + scheduler_specific_kwargs: + min_lr: 5.0e-07 + freeze_modules: 'qwen_vl_interface,layer_qformer,dino_encoder,dino_pro' # FIXME + loss_scale: + vla: 1.0 + vlm: 0.1 + pretrained_checkpoint: /hfm/cache/checkpoints/InternVLA-M1-Pretrain-RT-1-Bridge/checkpoints/steps_50000_pytorch_model.pt + # reload_modules: 'action_model,layer_qformer' # warning: this may cause model to not converge + skip_reload_modules: 'action_model' # skip action-related layers # FIXME + repeated_diffusion_steps: 4 + max_grad_norm: 1.0 + warmup_ratio: 0.1 + weight_decay: 0.0 + logging_frequency: 10 + gradient_clipping: 1.0 + gradient_accumulation_steps: 1 + + optimizer: + name: AdamW + betas: [0.9, 0.95] + eps: 1.0e-08 + weight_decay: 1.0e-08 + + # parameters to be determined + is_resume: false + resume_epoch: null + resume_step: null + enable_gradient_checkpointing: true + enable_mixed_precision_training: true diff --git a/src/InternVLA-M1/InternVLA/config/training/internvla_cotrain_libero.yaml b/src/InternVLA-M1/InternVLA/config/training/internvla_cotrain_libero.yaml new file mode 100644 index 0000000000000000000000000000000000000000..bd86afec1d27907cb6971c9fcf20e39ad6a6c407 --- /dev/null +++ b/src/InternVLA-M1/InternVLA/config/training/internvla_cotrain_libero.yaml @@ -0,0 +1,126 @@ +run_id: internVLA +run_root_dir: results/Checkpoints +seed: 42 +trackers: [jsonl, wandb] +wandb_entity: your_wandb_entity +wandb_project: internVLA +is_debug: false + + +framework: + framework_py: InternVLA-M1 + qwenvl: + base_vlm: ./playground/Pretrained_models/Qwen2.5-VL-3B-Instruct + attn_implementation: flash_attention_2 + vl_hidden_dim: 2048 + dino: + dino_backbone: dinov2_vits14 + layer_qformer: + qformer_end_layer: 37 + qformer_start_layer: 36 + num_query_tokens: 64 + input_dim: 2048 + ouptput_dim: 768 + grad_scale: 0.5 + action_model: + action_model_type: DiT-B + action_hidden_dim: 768 + action_dim: 7 + use_ema: false + future_action_window_size: 15 + past_action_window_size: 0 + repeated_diffusion_steps: 8 + + fm_head_config: + input_embedding_dim: 1536 # Action embedding after dimension 48 * 32,and DiT's attention_head_dim * attention_heads corresponds to ActionEncoder + hidden_size: 1024 # and DiT's last projection corresponds to ActionDecoder + add_pos_embed: True + max_seq_len: 1024 + action_dim: 7 + future_action_window_size: 15 + action_horizon: 16 + past_action_window_size: 0 + noise_beta_alpha: 1.5 + noise_beta_beta: 1.0 + noise_s: 0.999 + num_timestep_buckets: 1000 + num_inference_timesteps: 4 + num_target_vision_tokens: 32 + diffusion_model_cfg: # DiT Transformers parameters + attention_head_dim: 48 + cross_attention_dim: 2048 # VLM dim + dropout: 0.2 + final_dropout: true + interleave_self_attention: true + norm_type: "ada_norm" + num_attention_heads: 32 + num_layers: 16 + output_dim: 1024 + positional_embeddings: null + + reduce_in_full_precision: true + + +datasets: + vlm_data: + dataset_py: vlm_datasets + dataformat: llava_json + dataset_use: asv2_conversation_en,asv2_detailed_description_en,asv2_region_captioning_en,coco_internvl_longcap_en,coco_karpathy_train_567_en,coco_negative_gpt4o_en,coco_poetry_zh,coco_rem_en_zh,cocorem_exist_yorn_en,cocotextv2_en,cocotextv2_gpt4o_en,okvqa_en,refcoco_grounding_aug_en,refcoco_grounding_en,tallyqa_coco_en,toloka_grounding_aug_en,vqav2_en,vsr_en + eval_dataset: aokvqa_cauldron_llava_format + data_flatten: false + base_interval: 2 + max_pixels: 12845056 + min_pixels: 3136 + model_max_length: 2048 + model_type: qwen2.5vl + per_device_batch_size: 4 + + vla_data: + dataset_py: lerobot_datasets + data_root_dir: playground/Datasets/LEROBOT_LIBERO_DATA + data_mix: libero_goal # libero_goal + action_type: delta_qpos + CoT_prompt: "Your task is {instruction}. To identify the key objects for your task. Locate their bounding boxes in [x1,y1,x2,y2] format." + CoT_answer: bbox + default_image_resolution: [3, 224, 224] + per_device_batch_size: 16 + load_all_data_for_training: true + obs: ["image_0"] + + +trainer: + epochs: 100 + max_train_steps: 100000 + num_warmup_steps: 5000 + save_interval: 5000 + eval_interval: 100 + learning_rate: + base: 2.5e-05 + qwen_vl_interface: 1.0e-05 + action_model: 1.0e-04 + lr_scheduler_type: cosine_with_min_lr + scheduler_specific_kwargs: + min_lr: 1.0e-06 + freeze_modules: '' + loss_scale: + vla: 1.0 + vlm: 0.1 + max_grad_norm: 1.0 + warmup_ratio: 0.1 + weight_decay: 0.0 + logging_frequency: 10 + gradient_clipping: 1.0 + gradient_accumulation_steps: 1 + + optimizer: + name: AdamW + betas: [0.9, 0.95] + eps: 1.0e-08 + weight_decay: 1.0e-08 + + # parameters to be determined + is_resume: false + resume_epoch: null + resume_step: null + enable_gradient_checkpointing: true + enable_mixed_precision_training: true diff --git a/src/InternVLA-M1/InternVLA/config/training/internvla_cotrain_oxe.yaml b/src/InternVLA-M1/InternVLA/config/training/internvla_cotrain_oxe.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a982a269bcf3793f090be9b083ab48425f6363e5 --- /dev/null +++ b/src/InternVLA-M1/InternVLA/config/training/internvla_cotrain_oxe.yaml @@ -0,0 +1,100 @@ +run_id: qwenact +run_root_dir: results/Checkpoints +seed: 42 +trackers: [jsonl, wandb] +wandb_entity: your_wandb_entity +wandb_project: internVLAAA +is_debug: false + +framework: + framework_py: InternVLA-M1 + qwenvl: + base_vlm: ./playground/Pretrained_models/Qwen2.5-VL-3B-Instruct + attn_implementation: flash_attention_2 + vl_hidden_dim: 2048 + dino: + dino_backbone: dinov2_vits14 + layer_qformer: + qformer_end_layer: 37 + qformer_start_layer: 36 + num_query_tokens: 64 + input_dim: 2048 + ouptput_dim: 768 + grad_scale: 0.5 + + action_model: + action_model_type: DiT-B + action_hidden_dim: 768 + action_dim: 7 + use_ema: false + future_action_window_size: 15 + past_action_window_size: 0 + repeated_diffusion_steps: 8 + + +datasets: + vlm_data: + dataset_py: vlm_datasets + dataformat: llava_json + dataset_use: asv2_conversation_en,asv2_detailed_description_en,asv2_region_captioning_en,coco_internvl_longcap_en,coco_karpathy_train_567_en,coco_negative_gpt4o_en,coco_poetry_zh,coco_rem_en_zh,cocorem_exist_yorn_en,cocotextv2_en,cocotextv2_gpt4o_en,okvqa_en,refcoco_grounding_aug_en,refcoco_grounding_en,tallyqa_coco_en,toloka_grounding_aug_en,vqav2_en,vsr_en + eval_dataset: aokvqa_cauldron_llava_format + data_flatten: false + base_interval: 2 + max_pixels: 50176 + min_pixels: 784 + model_max_length: 2048 + model_type: qwen2.5vl + per_device_batch_size: 4 + + vla_data: + dataset_py: lerobot_datasets + data_root_dir: playground/Datasets/OXE_LEROBOT_DATASET + data_mix: bridge_rt_1 # bridge_rt_1 + action_type: delta_ee + CoT_prompt: "Your task is {instruction}. To identify the key objects for your task. Locate their bounding boxes in [x1,y1,x2,y2] format." + CoT_answer: bbox + default_image_resolution: [3, 224, 224] + per_device_batch_size: 16 + load_all_data_for_training: true + obs: ["image_0"] + image_size: [224,224] + +trainer: + epochs: 100 + max_train_steps: 100000 + num_warmup_steps: 5000 + save_interval: 5000 + eval_interval: 100 + learning_rate: + base: 1e-05 + qwen_vl_interface: 1.0e-05 + action_model: 1.0e-04 + lr_scheduler_type: cosine_with_min_lr + scheduler_specific_kwargs: + min_lr: 5.0e-07 + freeze_modules: '' + loss_scale: + vla: 1.0 + vlm: 0.1 + # pretrained_checkpoint: ./results/Checkpoints/1_need/0601_qwenact_fixqwen_32gpus_lr_1e-3_qformer_36_37/steps_100000/pytorch_model.pt + # reload_modules: 'action_model,layer_qformer' # warning: this may cause model to not converge + repeated_diffusion_steps: 4 + max_grad_norm: 1.0 + warmup_ratio: 0.1 + weight_decay: 0.0 + logging_frequency: 10 + gradient_clipping: 1.0 + gradient_accumulation_steps: 1 + + optimizer: + name: AdamW + betas: [0.9, 0.95] + eps: 1.0e-08 + weight_decay: 1.0e-08 + + # parameters to be determined + is_resume: false + resume_epoch: null + resume_step: null + enable_gradient_checkpointing: true + enable_mixed_precision_training: true diff --git a/src/InternVLA-M1/InternVLA/config/training/internvla_cotrain_sim_demo.yaml b/src/InternVLA-M1/InternVLA/config/training/internvla_cotrain_sim_demo.yaml new file mode 100644 index 0000000000000000000000000000000000000000..48ae72ab8f30b98f492bfbf8b495ee795338efff --- /dev/null +++ b/src/InternVLA-M1/InternVLA/config/training/internvla_cotrain_sim_demo.yaml @@ -0,0 +1,124 @@ +run_id: qwenact +run_root_dir: results/Checkpoints +seed: 42 +trackers: [jsonl, wandb] +wandb_entity: ailab-manipulation +wandb_project: internVLA +is_debug: false + +framework: + framework_py: InternVLA-M1 + qwenvl: # TODO later should not use qwenvl should use vlm directly + base_vlm: ./playground/Pretrained_models/Qwen2.5-VL-3B-Instruct + attn_implementation: flash_attention_2 + vl_hidden_dim: 2048 # TODO should read from internal config + dino: + dino_backbone: dinov2_vits14 + layer_qformer: + qformer_end_layer: 37 + qformer_start_layer: 36 + num_query_tokens: 64 + input_dim: 2048 # here is VLM output dimension + ouptput_dim: 768 # here is action input dimension # here has a rule that ouptput_dim = action_hidden_dim; --> top2down logic should not default match, should be modified manually + grad_scale: 0.5 # let gradient pass through this module to decay, avoid destroying VLM, # seems to affect learning + + action_model: + action_model_type: DiT-B + action_hidden_dim: 768 + action_dim: 8 + use_ema: false + future_action_window_size: 15 + past_action_window_size: 0 + repeated_diffusion_steps: 8 + + fm_head_config: + input_embedding_dim: 1536 # Action embedding after dimension 48 * 32,and DiT's attention_head_dim * attention_heads corresponds to ActionEncoder + hidden_size: 1024 # and DiT's last projection corresponds to ActionDecoder + add_pos_embed: True + max_seq_len: 1024 + action_dim: 8 + future_action_window_size: 15 + action_horizon: 16 # = future_action_window_size + 1 + past_action_window_size + past_action_window_size: 0 + noise_beta_alpha: 1.5 + noise_beta_beta: 1.0 + noise_s: 0.999 + num_timestep_buckets: 1000 + num_inference_timesteps: 4 + num_target_vision_tokens: 32 + diffusion_model_cfg: # DiT Transformers parameters + attention_head_dim: 48 + cross_attention_dim: 2048 # VLM dim + dropout: 0.2 + final_dropout: true + interleave_self_attention: true + norm_type: "ada_norm" + num_attention_heads: 32 + num_layers: 16 + output_dim: 1024 + positional_embeddings: null + +datasets: + vlm_data: + dataset_py: vlm_datasets + dataformat: llava_json + dataset_use: asv2_conversation_en,asv2_detailed_description_en,asv2_region_captioning_en,coco_internvl_longcap_en,coco_karpathy_train_567_en,coco_negative_gpt4o_en,coco_poetry_zh,coco_rem_en_zh,cocorem_exist_yorn_en,cocotextv2_en,cocotextv2_gpt4o_en,okvqa_en,refcoco_grounding_aug_en,refcoco_grounding_en,tallyqa_coco_en,toloka_grounding_aug_en,vqav2_en,vsr_en + eval_dataset: aokvqa_cauldron_llava_format + data_flatten: false + base_interval: 2 + max_pixels: 50176 + min_pixels: 784 + model_max_length: 2048 # seems to be 4096 here, some related content is not processed + model_type: qwen2.5vl + per_device_batch_size: 4 + + vla_data: + dataset_py: lerobot_datasets + data_root_dir: playground/demo_data + data_mix: demo_sim_pick_place + action_type: delta_joints + CoT_prompt: "Your task is {instruction}. To identify the key objects for your task. Locate their bounding boxes in [x1,y1,x2,y2] format." + CoT_answer: bbox + default_image_resolution: [3, 224, 224] # this is not effective + per_device_batch_size: 16 + load_all_data_for_training: true # consider how to enable eval set + obs: ["image_0"] # this is not effective + image_size: [224,224] #todo is null keep raw size # this is not effective + +trainer: + epochs: 100 # decide whether to use, he and max_train_steps conflict + max_train_steps: 100000 + num_warmup_steps: 5000 + save_interval: 5000 + eval_interval: 100 + learning_rate: # you can set different lr for different modules + base: 5e-05 + qwen_vl_interface: 1.0e-05 + action_model: 1.0e-04 + lr_scheduler_type: cosine_with_min_lr + scheduler_specific_kwargs: + min_lr: 5.0e-07 + freeze_modules: '' + loss_scale: + vla: 1.0 + vlm: 0.1 + repeated_diffusion_steps: 4 + max_grad_norm: 1.0 + warmup_ratio: 0.1 + weight_decay: 0.0 + logging_frequency: 10 + gradient_clipping: 1.0 + gradient_accumulation_steps: 1 + + optimizer: + name: AdamW + betas: [0.9, 0.95] + eps: 1.0e-08 + weight_decay: 1.0e-08 + + # parameters to be determined + is_resume: false + resume_epoch: null + resume_step: null + enable_gradient_checkpointing: true + enable_mixed_precision_training: true diff --git a/src/InternVLA-M1/InternVLA/dataloader/__init__.py b/src/InternVLA-M1/InternVLA/dataloader/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..4586148a8e60c120c29fdf7fc780bf947a530af6 --- /dev/null +++ b/src/InternVLA-M1/InternVLA/dataloader/__init__.py @@ -0,0 +1,63 @@ +import json +import os +from accelerate.logging import get_logger +import numpy as np +from torch.utils.data import DataLoader +import numpy as np +import torch.distributed as dist +from pathlib import Path +from InternVLA.dataloader.vlm_datasets import make_vlm_dataloader + +logger = get_logger(__name__) + +def save_dataset_statistics(dataset_statistics, run_dir): + """Saves a `dataset_statistics.json` file.""" + out_path = run_dir / "dataset_statistics.json" + with open(out_path, "w") as f_json: + for _, stats in dataset_statistics.items(): + for k in stats["action"].keys(): + if isinstance(stats["action"][k], np.ndarray): + stats["action"][k] = stats["action"][k].tolist() + if "proprio" in stats: + for k in stats["proprio"].keys(): + if isinstance(stats["proprio"][k], np.ndarray): + stats["proprio"][k] = stats["proprio"][k].tolist() + if "num_trajectories" in stats: + if isinstance(stats["num_trajectories"], np.ndarray): + stats["num_trajectories"] = stats["num_trajectories"].item() + if "num_transitions" in stats: + if isinstance(stats["num_transitions"], np.ndarray): + stats["num_transitions"] = stats["num_transitions"].item() + json.dump(dataset_statistics, f_json, indent=2) + logger.info(f"Saved dataset statistics file at path {out_path}") + + + +def build_dataloader(cfg, dataset_py="lerobot_datasets_oxe"): # TODO now here only is get dataset, we need mv dataloader to here + + if dataset_py == "lerobot_datasets": + from InternVLA.dataloader.lerobot_datasets import get_vla_dataset, collate_fn + vla_dataset_cfg = cfg.datasets.vla_data + + data_root_dir = vla_dataset_cfg.data_root_dir + data_mix = vla_dataset_cfg.data_mix + + vla_dataset = get_vla_dataset(data_cfg=vla_dataset_cfg) + + vla_train_dataloader = DataLoader( + vla_dataset, + batch_size=cfg.datasets.vla_data.per_device_batch_size, + collate_fn=collate_fn, + num_workers=8, + # shuffle=True + ) + if dist.get_rank() == 0: + + output_dir = Path(cfg.output_dir) + vla_dataset.save_dataset_statistics(output_dir / "dataset_statistics.json") + return vla_train_dataloader + elif dataset_py == "vlm_datasets": + vlm_data_module = make_vlm_dataloader(cfg) + vlm_train_dataloader = vlm_data_module["train_dataloader"] + + return vlm_train_dataloader diff --git a/src/InternVLA-M1/InternVLA/dataloader/gr00t_lerobot/README.md b/src/InternVLA-M1/InternVLA/dataloader/gr00t_lerobot/README.md new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/src/InternVLA-M1/InternVLA/dataloader/gr00t_lerobot/__init__.py b/src/InternVLA-M1/InternVLA/dataloader/gr00t_lerobot/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/src/InternVLA-M1/InternVLA/dataloader/gr00t_lerobot/data_config.py b/src/InternVLA-M1/InternVLA/dataloader/gr00t_lerobot/data_config.py new file mode 100644 index 0000000000000000000000000000000000000000..2d82e3499af6515de483da1e47f3407b9b0290ca --- /dev/null +++ b/src/InternVLA-M1/InternVLA/dataloader/gr00t_lerobot/data_config.py @@ -0,0 +1,770 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from abc import ABC, abstractmethod + +from InternVLA.dataloader.gr00t_lerobot.datasets import ModalityConfig +from InternVLA.dataloader.gr00t_lerobot.transform.base import ComposedModalityTransform, ModalityTransform +from InternVLA.dataloader.gr00t_lerobot.transform.concat import ConcatTransform +from InternVLA.dataloader.gr00t_lerobot.transform.state_action import ( + StateActionSinCosTransform, + StateActionToTensor, + StateActionTransform, +) +from InternVLA.dataloader.gr00t_lerobot.transform.video import ( + VideoColorJitter, + VideoCrop, + VideoResize, + VideoToNumpy, + VideoToTensor, +) +# from gr00t.model.transforms import GR00TTransform + + +class BaseDataConfig(ABC): + @abstractmethod + def modality_config(self) -> dict[str, ModalityConfig]: + pass + + @abstractmethod + def transform(self) -> ModalityTransform: + pass + + +########################################################################################### + +class OxeDroidDataConfig: + video_keys = [ + "video.exterior_image_1", + "video.exterior_image_2", + "video.wrist_image", + ] + state_keys = [ + "state.eef_position", + "state.eef_rotation", + "state.gripper_position", + ] + action_keys = [ + "action.eef_position_delta", + "action.eef_rotation_delta", + "action.gripper_position", + ] + language_keys = ["annotation.language.language_instruction"] + observation_indices = [0] + action_indices = list(range(16)) + + def modality_config(self): + video_modality = ModalityConfig( + delta_indices=self.observation_indices, + modality_keys=self.video_keys, + ) + state_modality = ModalityConfig( + delta_indices=self.observation_indices, + modality_keys=self.state_keys, + ) + action_modality = ModalityConfig( + delta_indices=self.action_indices, + modality_keys=self.action_keys, + ) + language_modality = ModalityConfig( + delta_indices=self.observation_indices, + modality_keys=self.language_keys, + ) + modality_configs = { + "video": video_modality, + "state": state_modality, + "action": action_modality, + "language": language_modality, + } + return modality_configs + + def transform(self): + transforms = [ + # video transforms + VideoToTensor(apply_to=self.video_keys), + VideoCrop(apply_to=self.video_keys, scale=0.95), + VideoResize(apply_to=self.video_keys, height=224, width=224, interpolation="linear"), + VideoColorJitter( + apply_to=self.video_keys, + brightness=0.3, + contrast=0.4, + saturation=0.5, + hue=0.08, + ), + VideoToNumpy(apply_to=self.video_keys), + # state transforms + StateActionToTensor(apply_to=self.state_keys), + StateActionTransform( + apply_to=self.state_keys, + normalization_modes={ + "state.eef_position": "min_max", + "state.gripper_position": "min_max", + }, + target_rotations={ + "state.eef_rotation": "rotation_6d", + }, + ), + # action transforms + StateActionToTensor(apply_to=self.action_keys), + StateActionTransform( + apply_to=self.action_keys, + normalization_modes={ + "action.gripper_position": "binary", + }, + target_rotations={"action.eef_rotation_delta": "axis_angle"}, + ), + # concat transforms + ConcatTransform( + video_concat_order=self.video_keys, + state_concat_order=self.state_keys, + action_concat_order=self.action_keys, + ), + GR00TTransform( + state_horizon=len(self.observation_indices), + action_horizon=len(self.action_indices), + max_state_dim=64, + max_action_dim=32, + ), + ] + + return ComposedModalityTransform(transforms=transforms) + + +########################################################################################### + + +class OxeBridgeDataConfig: + video_keys = [ + "video.image_0", + ] + state_keys = [ + "state.x", + "state.y", + "state.z", + "state.roll", + "state.pitch", + "state.yaw", + "state.pad", + "state.gripper", + ] + action_keys = [ + "action.x", + "action.y", + "action.z", + "action.roll", + "action.pitch", + "action.yaw", + "action.gripper", + ] + language_keys = ["annotation.human.action.task_description"] + observation_indices = [0] + action_indices = list(range(16)) + + def modality_config(self): + video_modality = ModalityConfig( + delta_indices=self.observation_indices, + modality_keys=self.video_keys, + ) + state_modality = ModalityConfig( + delta_indices=self.observation_indices, + modality_keys=self.state_keys, + ) + action_modality = ModalityConfig( + delta_indices=self.action_indices, + modality_keys=self.action_keys, + ) + language_modality = ModalityConfig( + delta_indices=self.observation_indices, + modality_keys=self.language_keys, + ) + modality_configs = { + "video": video_modality, + "state": state_modality, + "action": action_modality, + "language": language_modality, + } + return modality_configs + + def transform(self): + transforms = [ + # video transforms + # VideoToTensor(apply_to=self.video_keys), + # VideoCrop(apply_to=self.video_keys, scale=0.95), + # VideoResize(apply_to=self.video_keys, height=224, width=224, interpolation="linear"), + # VideoColorJitter( + # apply_to=self.video_keys, + # brightness=0.3, + # contrast=0.4, + # saturation=0.5, + # hue=0.08, + # ), + # VideoToNumpy(apply_to=self.video_keys), + # state transforms + StateActionToTensor(apply_to=self.state_keys), + StateActionTransform( + apply_to=self.state_keys, + normalization_modes={ + "state.x": "q99", + "state.y": "q99", + "state.z": "q99", + "state.roll": "q99", + "state.pitch": "q99", + "state.yaw": "q99", + "state.pad": "q99", + "state.gripper": "binary", + }, + ), + # action transforms + StateActionToTensor(apply_to=self.action_keys), + StateActionTransform( + apply_to=self.action_keys, + normalization_modes={ + "action.x": "q99", + "action.y": "q99", + "action.z": "q99", + "action.roll": "q99", + "action.pitch": "q99", + "action.yaw": "q99", + "action.gripper": "binary", + }, + ), + # concat transforms + # ConcatTransform( + # # video_concat_order=self.video_keys, + # state_concat_order=self.state_keys, + # action_concat_order=self.action_keys, + # ), + # GR00TTransform( + # state_horizon=len(self.observation_indices), + # action_horizon=len(self.action_indices), + # max_state_dim=64, + # max_action_dim=32, + # ), + ] + + return ComposedModalityTransform(transforms=transforms) + + +########################################################################################### + +class OxeRT1DataConfig: + video_keys = [ + "video.image", + ] + state_keys = [ + "state.x", + "state.y", + "state.z", + "state.rx", + "state.ry", + "state.rz", + "state.rw", + "state.gripper", + ] + action_keys = [ + "action.x", + "action.y", + "action.z", + "action.roll", + "action.pitch", + "action.yaw", + "action.gripper", + ] + language_keys = ["annotation.human.action.task_description"] + observation_indices = [0] + action_indices = list(range(16)) + + def modality_config(self): + video_modality = ModalityConfig( + delta_indices=self.observation_indices, + modality_keys=self.video_keys, + ) + state_modality = ModalityConfig( + delta_indices=self.observation_indices, + modality_keys=self.state_keys, + ) + action_modality = ModalityConfig( + delta_indices=self.action_indices, + modality_keys=self.action_keys, + ) + language_modality = ModalityConfig( + delta_indices=self.observation_indices, + modality_keys=self.language_keys, + ) + modality_configs = { + "video": video_modality, + "state": state_modality, + "action": action_modality, + "language": language_modality, + } + return modality_configs + + def transform(self): + transforms = [ + # video transforms + # VideoToTensor(apply_to=self.video_keys), + # VideoCrop(apply_to=self.video_keys, scale=0.95), + # VideoResize(apply_to=self.video_keys, height=224, width=224, interpolation="linear"), + # VideoColorJitter( + # apply_to=self.video_keys, + # brightness=0.3, + # contrast=0.4, + # saturation=0.5, + # hue=0.08, + # ), + # VideoToNumpy(apply_to=self.video_keys), + # state transforms + StateActionToTensor(apply_to=self.state_keys), + StateActionTransform( + apply_to=self.state_keys, + normalization_modes={ + "state.x": "q99", + "state.y": "q99", + "state.z": "q99", + "state.rx": "q99", + "state.ry": "q99", + "state.rz": "q99", + "state.rw": "q99", + "state.gripper": "binary", + }, + ), + # action transforms + StateActionToTensor(apply_to=self.action_keys), + StateActionTransform( + apply_to=self.action_keys, + normalization_modes={ + "action.x": "q99", + "action.y": "q99", + "action.z": "q99", + "action.roll": "q99", + "action.pitch": "q99", + "action.yaw": "q99", + "action.gripper": "binary", + }, + ), + # concat transforms + # ConcatTransform( + # # video_concat_order=self.video_keys, + # state_concat_order=self.state_keys, + # action_concat_order=self.action_keys, + # ), + # GR00TTransform( + # state_horizon=len(self.observation_indices), + # action_horizon=len(self.action_indices), + # max_state_dim=64, + # max_action_dim=32, + # ), + ] + + return ComposedModalityTransform(transforms=transforms) + + +########################################################################################### + + +class SingleFrankaRobotiqDeltaEefDataConfig: + video_keys = [ + "video.base_view", + "video.ego_view", + ] + state_keys = [ + "state.eef_position", + "state.eef_rotation", + ] + action_keys = [ + "action.delta_eef_position", + "action.delta_eef_rotation", + "action.gripper_close", + ] + + language_keys = ["annotation.human.action.task_description"] + observation_indices = [0] + action_indices = list(range(16)) + + def modality_config(self): + video_modality = ModalityConfig( + delta_indices=self.observation_indices, + modality_keys=self.video_keys, + ) + state_modality = ModalityConfig( + delta_indices=self.observation_indices, + modality_keys=self.state_keys, + ) + action_modality = ModalityConfig( + delta_indices=self.action_indices, + modality_keys=self.action_keys, + ) + language_modality = ModalityConfig( + delta_indices=self.observation_indices, + modality_keys=self.language_keys, + ) + modality_configs = { + "video": video_modality, + "state": state_modality, + "action": action_modality, + "language": language_modality, + } + return modality_configs + + def transform(self): + transforms = [ + # state transforms + StateActionToTensor(apply_to=self.state_keys), + StateActionTransform( + apply_to=self.state_keys, + normalization_modes={ + "state.eef_position": "min_max", + "state.eef_rotation": "min_max", + }, + ), + # action transforms + StateActionToTensor(apply_to=self.action_keys), + StateActionTransform( + apply_to=self.action_keys, + normalization_modes={ + "action.delta_eef_position": "min_max", + "action.delta_eef_rotation": "min_max", + "action.gripper_close": "binary", + }, + ), + ] + + return ComposedModalityTransform(transforms=transforms) + +########################################################################################### + +class Libero4in1DataConfig: + video_keys = [ + "video.primary_image", + "video.wrist_image", + ] + + state_keys = [ + "state.x", + "state.y", + "state.z", + "state.roll", + "state.pitch", + "state.yaw", + "state.pad", + "state.gripper", + ] + action_keys = [ + "action.x", + "action.y", + "action.z", + "action.roll", + "action.pitch", + "action.yaw", + "action.gripper", + ] + + language_keys = ["annotation.human.action.task_description"] + + observation_indices = [0] + action_indices = list(range(8)) + + + def modality_config(self): + video_modality = ModalityConfig( + delta_indices=self.observation_indices, + modality_keys=self.video_keys, + ) + state_modality = ModalityConfig( + delta_indices=self.observation_indices, + modality_keys=self.state_keys, + ) + action_modality = ModalityConfig( + delta_indices=self.action_indices, + modality_keys=self.action_keys, + ) + language_modality = ModalityConfig( + delta_indices=self.observation_indices, + modality_keys=self.language_keys, + ) + modality_configs = { + "video": video_modality, + "state": state_modality, + "action": action_modality, + "language": language_modality, + } + return modality_configs + + def transform(self): + transforms = [ + # action transforms + StateActionToTensor(apply_to=self.action_keys), + StateActionTransform( + apply_to=self.action_keys, + normalization_modes={ + "action.x": "min_max", + "action.y": "min_max", + "action.z": "min_max", + "action.roll": "min_max", + "action.pitch": "min_max", + "action.yaw": "min_max", + }, + ), + ] + + return ComposedModalityTransform(transforms=transforms) + +########################################################################################### + + +class SingleFrankaRobotiqDeltaJointsDataConfig: + video_keys = [ + "video.base_view", + "video.ego_view", + ] + state_keys = [ + "state.joints", + ] + action_keys = [ + "action.delta_joints", + "action.gripper_close", + ] + + language_keys = ["annotation.human.action.task_description"] + observation_indices = [0] + action_indices = list(range(16)) + + def modality_config(self): + video_modality = ModalityConfig( + delta_indices=self.observation_indices, + modality_keys=self.video_keys, + ) + state_modality = ModalityConfig( + delta_indices=self.observation_indices, + modality_keys=self.state_keys, + ) + action_modality = ModalityConfig( + delta_indices=self.action_indices, + modality_keys=self.action_keys, + ) + language_modality = ModalityConfig( + delta_indices=self.observation_indices, + modality_keys=self.language_keys, + ) + modality_configs = { + "video": video_modality, + "state": state_modality, + "action": action_modality, + "language": language_modality, + } + return modality_configs + + def transform(self): + transforms = [ + # state transforms + StateActionToTensor(apply_to=self.state_keys), + StateActionTransform( + apply_to=self.state_keys, + normalization_modes={ + "state.joints": "min_max", + }, + ), + # action transforms + StateActionToTensor(apply_to=self.action_keys), + StateActionTransform( + apply_to=self.action_keys, + normalization_modes={ + "action.delta_joints": "min_max", + "action.gripper_close": "binary", + }, + ), + ] + + return ComposedModalityTransform(transforms=transforms) + +class HumanoidDataConfig: + video_keys = [ + "video.rs_view", + ] + state_keys = [ + "state.left_hand", + "state.right_hand", + "state.left_arm", + "state.right_arm", + "state.rpy", + "state.height", + ] + action_keys = [ + "action.left_hand", + "action.right_hand", + "action.left_arm", + "action.right_arm", + "action.rpy", + "action.height", + "action.torso_vx", + "action.torso_vy", + "action.torso_vyaw", + "action.target_yaw", + ] + + language_keys = ["annotation.human.task_description"] + observation_indices = [0] + action_indices = list(range(16)) # list(range(30)) + + def modality_config(self): + video_modality = ModalityConfig( + delta_indices=self.observation_indices, + modality_keys=self.video_keys, + ) + state_modality = ModalityConfig( + delta_indices=self.observation_indices, + modality_keys=self.state_keys, + ) + action_modality = ModalityConfig( + delta_indices=self.action_indices, + modality_keys=self.action_keys, + ) + language_modality = ModalityConfig( + delta_indices=self.observation_indices, + modality_keys=self.language_keys, + ) + modality_configs = { + "video": video_modality, + "state": state_modality, + "action": action_modality, + "language": language_modality, + } + return modality_configs + + def transform(self): + transforms = [ + # state transforms + StateActionToTensor(apply_to=self.state_keys), + StateActionTransform( + apply_to=self.state_keys, + normalization_modes={ + "state.left_hand": "min_max", + "state.right_hand": "min_max", + "state.left_arm": "min_max", + "state.right_arm": "min_max", + "state.rpy": "min_max", + "state.height": "min_max", + }, + ), + # action transforms + StateActionToTensor(apply_to=self.action_keys), + StateActionTransform( + apply_to=self.action_keys, + normalization_modes={ + "action.left_hand": "min_max", + "action.right_hand": "min_max", + "action.left_arm": "min_max", + "action.right_arm": "min_max", + "action.rpy": "min_max", + "action.height": "min_max", + "action.torso_vx": "min_max", + "action.torso_vy": "min_max", + "action.torso_vyaw": "min_max", + "action.target_yaw": "min_max", + }, + ), + ] + + return ComposedModalityTransform(transforms=transforms) + + +class HumanoidSimDataConfig: + video_keys = ["video.rs_view"] + state_keys = [ + "state.left_hand", + "state.right_hand", + "state.left_arm", + "state.right_arm", + "state.waist", + ] + action_keys = [ + "action.left_hand", + "action.right_hand", + "action.left_arm", + "action.right_arm", + "action.waist", + ] + language_keys = ["annotation.human.task_description"] + observation_indices = [0] + action_indices = list(range(16)) + + def modality_config(self): + video_modality = ModalityConfig( + delta_indices=self.observation_indices, + modality_keys=self.video_keys, + ) + state_modality = ModalityConfig( + delta_indices=self.observation_indices, + modality_keys=self.state_keys, + ) + action_modality = ModalityConfig( + delta_indices=self.action_indices, + modality_keys=self.action_keys, + ) + language_modality = ModalityConfig( + delta_indices=self.observation_indices, + modality_keys=self.language_keys, + ) + return { + "video": video_modality, + "state": state_modality, + "action": action_modality, + "language": language_modality, + } + + def transform(self): + transforms = [ + VideoToTensor(apply_to=self.video_keys), + VideoCrop(apply_to=self.video_keys, scale=0.95), + VideoResize(apply_to=self.video_keys, height=224, width=224, interpolation="linear"), + VideoColorJitter( + apply_to=self.video_keys, + brightness=0.3, + contrast=0.4, + saturation=0.5, + hue=0.08, + ), + VideoToNumpy(apply_to=self.video_keys), + StateActionToTensor(apply_to=self.state_keys), + StateActionTransform( + apply_to=self.state_keys, + normalization_modes={key: "min_max" for key in self.state_keys}, + ), + StateActionToTensor(apply_to=self.action_keys), + StateActionTransform( + apply_to=self.action_keys, + normalization_modes={key: "min_max" for key in self.action_keys}, + ), + ConcatTransform( + video_concat_order=self.video_keys, + state_concat_order=self.state_keys, + action_concat_order=self.action_keys, + ), + ] + return ComposedModalityTransform(transforms=transforms) +########################################################################################### + + + +ROBOT_TYPE_CONFIG_MAP = { + "libero_franka": Libero4in1DataConfig(), + "oxe_droid": OxeDroidDataConfig(), + "oxe_bridge": OxeBridgeDataConfig(), + "oxe_rt1": OxeRT1DataConfig(), + "demo_sim_franka_delta_joints": SingleFrankaRobotiqDeltaJointsDataConfig(), + "custom_robot_config": SingleFrankaRobotiqDeltaEefDataConfig(), + "humanoid_config": HumanoidDataConfig(), + "humanoid_sim_config": HumanoidSimDataConfig(), +} diff --git a/src/InternVLA-M1/InternVLA/dataloader/gr00t_lerobot/datasets.py b/src/InternVLA-M1/InternVLA/dataloader/gr00t_lerobot/datasets.py new file mode 100644 index 0000000000000000000000000000000000000000..95831da6a47e586a44da0e66833bd554e6527bc0 --- /dev/null +++ b/src/InternVLA-M1/InternVLA/dataloader/gr00t_lerobot/datasets.py @@ -0,0 +1,2118 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +""" +In this file, we define 3 types of datasets: +1. LeRobotSingleDataset: a single dataset for a given embodiment tag +2. LeRobotMixtureDataset: a mixture of datasets for a given list of embodiment tags +3. CachedLeRobotSingleDataset: a single dataset for a given embodiment tag, + with caching for the video frames + +See `scripts/load_dataset.py` for examples on how to use these datasets. +""" + +import hashlib +import json +from collections import defaultdict +from pathlib import Path +from typing import Sequence + +import numpy as np +import pandas as pd +from pydantic import BaseModel, Field, ValidationError +from torch.utils.data import Dataset +from tqdm import tqdm +from PIL import Image + +from InternVLA.dataloader.gr00t_lerobot.video import get_all_frames, get_frames_by_timestamps + +from InternVLA.dataloader.gr00t_lerobot.embodiment_tags import EmbodimentTag +from InternVLA.dataloader.gr00t_lerobot.schema import ( + DatasetMetadata, + DatasetStatisticalValues, + LeRobotModalityMetadata, + LeRobotStateActionMetadata, +) +from InternVLA.dataloader.gr00t_lerobot.transform import ComposedModalityTransform + +from functools import partial +from typing import Tuple, List +import pickle + +LE_ROBOT_MODALITY_FILENAME = "meta/modality.json" +LE_ROBOT_EPISODE_FILENAME = "meta/episodes.jsonl" +LE_ROBOT_TASKS_FILENAME = "meta/tasks.jsonl" +LE_ROBOT_INFO_FILENAME = "meta/info.json" +LE_ROBOT_STATS_FILENAME = "meta/stats_gr00t.json" +LE_ROBOT_DATA_FILENAME = "data/*/*.parquet" +LE_ROBOT_STEPS_FILENAME = "meta/steps.pkl" +EPSILON = 5e-4 + +def calculate_dataset_statistics(parquet_paths: list[Path]) -> dict: + """Calculate the dataset statistics of all columns for a list of parquet files.""" + # Dataset statistics + all_low_dim_data_list = [] + # Collect all the data + # parquet_paths = parquet_paths[:3] + for parquet_path in tqdm( + sorted(list(parquet_paths)), + desc="Collecting all parquet files...", + ): + # Load the parquet file + parquet_data = pd.read_parquet(parquet_path) + parquet_data = parquet_data + all_low_dim_data_list.append(parquet_data) + all_low_dim_data = pd.concat(all_low_dim_data_list, axis=0) + # Compute dataset statistics + dataset_statistics = {} + for le_modality in all_low_dim_data.columns: + if le_modality not in ["states", "action"]: + continue + if le_modality.startswith("annotation."): + continue + print(f"Computing statistics for {le_modality}...") + np_data = np.vstack( + [np.asarray(x, dtype=np.float32) for x in all_low_dim_data[le_modality]] + ) + dataset_statistics[le_modality] = { + "mean": np.mean(np_data, axis=0).tolist(), + "std": np.std(np_data, axis=0).tolist(), + "min": np.min(np_data, axis=0).tolist(), + "max": np.max(np_data, axis=0).tolist(), + "q01": np.quantile(np_data, 0.01, axis=0).tolist(), + "q99": np.quantile(np_data, 0.99, axis=0).tolist(), + } + return dataset_statistics + + +class ModalityConfig(BaseModel): + """Configuration for a modality.""" + + delta_indices: list[int] + """Delta indices to sample relative to the current index. The returned data will correspond to the original data at a sampled base index + delta indices.""" + modality_keys: list[str] + """The keys to load for the modality in the dataset.""" + + +class LeRobotSingleDataset(Dataset): + """ + Base dataset class for LeRobot that supports sharding. + """ + def __init__( + self, + dataset_path: Path | str, + modality_configs: dict[str, ModalityConfig], + embodiment_tag: str | EmbodimentTag, + video_backend: str = "decord", + video_backend_kwargs: dict | None = None, + transforms: ComposedModalityTransform | None = None, + delete_pause_frame: bool = False, + preload_all=True + ): + """ + Initialize the dataset. + + Args: + dataset_path (Path | str): The path to the dataset. + modality_configs (dict[str, ModalityConfig]): The configuration for each modality. The keys are the modality names, and the values are the modality configurations. + See `ModalityConfig` for more details. + video_backend (str): Backend for video reading. + video_backend_kwargs (dict): Keyword arguments for the video backend when initializing the video reader. + transforms (ComposedModalityTransform): The transforms to apply to the dataset. + embodiment_tag (EmbodimentTag): Overload the embodiment tag for the dataset. e.g. define it as "new_embodiment" + """ + # first check if the path directory exists + if not Path(dataset_path).exists(): + raise FileNotFoundError(f"Dataset path {dataset_path} does not exist") + + self.delete_pause_frame = delete_pause_frame + + self.modality_configs = modality_configs + self.video_backend = video_backend + self.video_backend_kwargs = video_backend_kwargs if video_backend_kwargs is not None else {} + self.transforms = ( + transforms if transforms is not None else ComposedModalityTransform(transforms=[]) + ) + + self._dataset_path = Path(dataset_path) + self._dataset_name = self._dataset_path.name + if isinstance(embodiment_tag, EmbodimentTag): + self.tag = embodiment_tag.value + else: + self.tag = embodiment_tag + + self._metadata = self._get_metadata(EmbodimentTag(self.tag)) + + # LeRobot-specific config + self._lerobot_modality_meta = self._get_lerobot_modality_meta() + self._lerobot_info_meta = self._get_lerobot_info_meta() + self._data_path_pattern = self._get_data_path_pattern() + self._video_path_pattern = self._get_video_path_pattern() + self._chunk_size = self._get_chunk_size() + self._tasks = self._get_tasks() + self.curr_traj_data = None + self.curr_traj_id = None + + self._trajectory_ids, self._trajectory_lengths = self._get_trajectories() + if preload_all: + self._preload_all_trajectories() + self.preload_all = True + else: + self.preload_all = False + self._modality_keys = self._get_modality_keys() + self._delta_indices = self._get_delta_indices() + self._all_steps = self._get_all_steps() + self.set_transforms_metadata(self.metadata) + self.set_epoch(0) + + print(f"Initialized dataset {self.dataset_name} with {embodiment_tag}") + + + # Check if the dataset is valid + self._check_integrity() + + @property + def dataset_path(self) -> Path: + """The path to the dataset that contains the METADATA_FILENAME file.""" + return self._dataset_path + + @property + def metadata(self) -> DatasetMetadata: + """The metadata for the dataset, loaded from metadata.json in the dataset directory""" + return self._metadata + + @property + def trajectory_ids(self) -> np.ndarray: + """The trajectory IDs in the dataset, stored as a 1D numpy array of strings.""" + return self._trajectory_ids + + @property + def trajectory_lengths(self) -> np.ndarray: + """The trajectory lengths in the dataset, stored as a 1D numpy array of integers. + The order of the lengths is the same as the order of the trajectory IDs. + """ + return self._trajectory_lengths + + @property + def all_steps(self) -> list[tuple[int, int]]: + """The trajectory IDs and base indices for all steps in the dataset. + Example: + self.trajectory_ids: [0, 1, 2] + self.trajectory_lengths: [3, 2, 4] + return: [ + ("traj_0", 0), ("traj_0", 1), ("traj_0", 2), + ("traj_1", 0), ("traj_1", 1), + ("traj_2", 0), ("traj_2", 1), ("traj_2", 2), ("traj_2", 3) + ] + """ + return self._all_steps + + @property + def modality_keys(self) -> dict: + """The modality keys for the dataset. The keys are the modality names, and the values are the keys for each modality. + + Example: { + "video": ["video.image_side_0", "video.image_side_1"], + "state": ["state.eef_position", "state.eef_rotation"], + "action": ["action.eef_position", "action.eef_rotation"], + "language": ["language.human.task"], + "timestamp": ["timestamp"], + "reward": ["reward"], + } + """ + return self._modality_keys + + @property + def delta_indices(self) -> dict[str, np.ndarray]: + """The delta indices for the dataset. The keys are the modality.key, and the values are the delta indices for each modality.key.""" + return self._delta_indices + + @property + def dataset_name(self) -> str: + """The name of the dataset.""" + return self._dataset_name + + @property + def lerobot_modality_meta(self) -> LeRobotModalityMetadata: + """The metadata for the LeRobot dataset.""" + return self._lerobot_modality_meta + + @property + def lerobot_info_meta(self) -> dict: + """The metadata for the LeRobot dataset.""" + return self._lerobot_info_meta + + @property + def data_path_pattern(self) -> str: + """The path pattern for the LeRobot dataset.""" + return self._data_path_pattern + + @property + def video_path_pattern(self) -> str: + """The path pattern for the LeRobot dataset.""" + return self._video_path_pattern + + @property + def chunk_size(self) -> int: + """The chunk size for the LeRobot dataset.""" + return self._chunk_size + + @property + def tasks(self) -> pd.DataFrame: + """The tasks for the dataset.""" + return self._tasks + + def _get_metadata(self, embodiment_tag: EmbodimentTag) -> DatasetMetadata: + """Get the metadata for the dataset. + + Returns: + dict: The metadata for the dataset. + """ + + # 1. Modality metadata + modality_meta_path = self.dataset_path / LE_ROBOT_MODALITY_FILENAME + assert ( + modality_meta_path.exists() + ), f"Please provide a {LE_ROBOT_MODALITY_FILENAME} file in {self.dataset_path}" + # 1.1. State and action modalities + simplified_modality_meta: dict[str, dict] = {} + with open(modality_meta_path, "r") as f: + le_modality_meta = LeRobotModalityMetadata.model_validate(json.load(f)) + for modality in ["state", "action"]: + simplified_modality_meta[modality] = {} + le_state_action_meta: dict[str, LeRobotStateActionMetadata] = getattr( + le_modality_meta, modality + ) + for subkey in le_state_action_meta: + state_action_dtype = np.dtype(le_state_action_meta[subkey].dtype) + if np.issubdtype(state_action_dtype, np.floating): + continuous = True + else: + continuous = False + simplified_modality_meta[modality][subkey] = { + "absolute": le_state_action_meta[subkey].absolute, + "rotation_type": le_state_action_meta[subkey].rotation_type, + "shape": [ + le_state_action_meta[subkey].end - le_state_action_meta[subkey].start + ], + "continuous": continuous, + } + + # 1.2. Video modalities + le_info_path = self.dataset_path / LE_ROBOT_INFO_FILENAME + assert ( + le_info_path.exists() + ), f"Please provide a {LE_ROBOT_INFO_FILENAME} file in {self.dataset_path}" + with open(le_info_path, "r") as f: + le_info = json.load(f) + simplified_modality_meta["video"] = {} + for new_key in le_modality_meta.video: + original_key = le_modality_meta.video[new_key].original_key + if original_key is None: + original_key = new_key + le_video_meta = le_info["features"][original_key] + height = le_video_meta["shape"][le_video_meta["names"].index("height")] + width = le_video_meta["shape"][le_video_meta["names"].index("width")] + # NOTE(FH): different lerobot dataset versions have different keys for the number of channels and fps + try: + channels = le_video_meta["shape"][le_video_meta["names"].index("channel")] + fps = le_video_meta["video_info"]["video.fps"] + except (ValueError, KeyError): + # channels = le_video_meta["shape"][le_video_meta["names"].index("channels")] + channels = le_video_meta["info"]["video.channels"] + fps = le_video_meta["info"]["video.fps"] + simplified_modality_meta["video"][new_key] = { + "resolution": [width, height], + "channels": channels, + "fps": fps, + } + + # 2. Dataset statistics + stats_path = self.dataset_path / LE_ROBOT_STATS_FILENAME + try: + with open(stats_path, "r") as f: + le_statistics = json.load(f) + for stat in le_statistics.values(): + DatasetStatisticalValues.model_validate(stat) + except (FileNotFoundError, ValidationError) as e: + print(f"Failed to load dataset statistics: {e}") + print(f"Calculating dataset statistics for {self.dataset_name}") + # Get all parquet files in the dataset paths + parquet_files = list((self.dataset_path).glob(LE_ROBOT_DATA_FILENAME)) + le_statistics = calculate_dataset_statistics(parquet_files) + with open(stats_path, "w") as f: + json.dump(le_statistics, f, indent=4) + dataset_statistics = {} + for our_modality in ["state", "action"]: + dataset_statistics[our_modality] = {} + for subkey in simplified_modality_meta[our_modality]: + dataset_statistics[our_modality][subkey] = {} + state_action_meta = le_modality_meta.get_key_meta(f"{our_modality}.{subkey}") + assert isinstance(state_action_meta, LeRobotStateActionMetadata) + le_modality = state_action_meta.original_key + for stat_name in le_statistics[le_modality]: + indices = np.arange( + state_action_meta.start, + state_action_meta.end, + ) + stat = np.array(le_statistics[le_modality][stat_name]) + dataset_statistics[our_modality][subkey][stat_name] = stat[indices].tolist() + + # 3. Full dataset metadata + metadata = DatasetMetadata( + statistics=dataset_statistics, # type: ignore + modalities=simplified_modality_meta, # type: ignore + embodiment_tag=embodiment_tag, + ) + + return metadata + + def _get_trajectories(self) -> tuple[np.ndarray, np.ndarray]: + """Get the trajectories in the dataset.""" + # Get trajectory lengths, IDs, and whitelist from dataset metadata + episode_path = self.dataset_path / LE_ROBOT_EPISODE_FILENAME + with open(episode_path, "r") as f: + episode_metadata = [json.loads(line) for line in f] + trajectory_ids = [] + trajectory_lengths = [] + for episode in episode_metadata: + trajectory_ids.append(episode["episode_index"]) + trajectory_lengths.append(episode["length"]) + return np.array(trajectory_ids), np.array(trajectory_lengths) + + def _get_all_steps(self) -> list[tuple[int, int]]: + """Get the trajectory IDs and base indices for all steps in the dataset. + + Returns: + list[tuple[str, int]]: A list of (trajectory_id, base_index) tuples. + """ + # Create a hash key based on configuration to ensure cache validity + config_key = self._get_steps_config_key() + + # Create a unique filename based on config_key + steps_filename = f"steps_{config_key}.pkl" + # @DUG + # fast get static steps @fangjing --> don't use hash to dynamic sample + steps_filename = "steps_7799a3080fbd.pkl" + + steps_path = self.dataset_path / "meta" / steps_filename + + # Try to load cached steps first + try: + if steps_path.exists(): + with open(steps_path, "rb") as f: + cached_data = pickle.load(f) + return cached_data["steps"] + # # Verify the cached data matches current configuration + # if cached_data.get("config_key") == config_key: + # print(f"Loading cached steps from {steps_path}") + # return cached_data["steps"] + # else: + # print("Cached steps configuration mismatch, recomputing...") + except (FileNotFoundError, pickle.PickleError, KeyError) as e: + print(f"Failed to load cached steps: {e}") + print("Computing steps from scratch...") + + # Compute steps using single process + all_steps = self._get_all_steps_single_process() + + # Cache the computed steps with unique filename + try: + cache_data = { + "config_key": config_key, + "steps": all_steps, + "num_trajectories": len(self.trajectory_ids), + "total_steps": len(all_steps), + "computed_timestamp": pd.Timestamp.now().isoformat(), + "delete_pause_frame": self.delete_pause_frame, + } + + # Ensure the meta directory exists + steps_path.parent.mkdir(parents=True, exist_ok=True) + + with open(steps_path, "wb") as f: + pickle.dump(cache_data, f, protocol=pickle.HIGHEST_PROTOCOL) + print(f"Cached steps saved to {steps_path}") + except Exception as e: + print(f"Failed to cache steps: {e}") + + return all_steps + + def _get_steps_config_key(self) -> str: + """Generate a configuration key for steps caching.""" + config_dict = { + "delete_pause_frame": self.delete_pause_frame, + "dataset_name": self.dataset_name, + } + # Create a hash of the configuration + config_str = str(sorted(config_dict.items())) + return hashlib.md5(config_str.encode()).hexdigest()[:12] # + + + def _get_all_steps_single_process(self) -> list[tuple[int, int]]: + """Original single-process implementation as fallback.""" + all_steps: list[tuple[int, int]] = [] + skipped_trajectories = 0 + processed_trajectories = 0 + + # Check if language modality is configured + has_language_modality = 'language' in self.modality_keys and len(self.modality_keys['language']) > 0 + + for trajectory_id, trajectory_length in tqdm(zip(self.trajectory_ids, self.trajectory_lengths), total=len(self.trajectory_ids), desc="Getting All Step"): + data = self.get_trajectory_data(trajectory_id) + trajectory_skipped = False + + # Check if trajectory has valid language instruction (if language modality is configured) + if has_language_modality: + self.curr_traj_data = data # Set current trajectory data for get_language to work + try: + language_instruction = self.get_language(trajectory_id, self.modality_keys['language'][0], 0) + print(language_instruction) + if not language_instruction or language_instruction[0] == "": + print(f"Skipping trajectory {trajectory_id} due to empty language instruction") + skipped_trajectories += 1 + trajectory_skipped = True + continue + except Exception as e: + print(f"Skipping trajectory {trajectory_id} due to language retrieval error: {e}") + skipped_trajectories += 1 + trajectory_skipped = True + continue + + if not trajectory_skipped: + processed_trajectories += 1 + + self.delete_pause_frame = False # FIXME + if self.delete_pause_frame: + # Get position and gripper fields based on available columns + delta_position_values, gripper_values = self._get_position_and_gripper_values(data) + previous_gripper = gripper_values[0] + for base_index in range(trajectory_length): + if base_index >= len(delta_position_values) or base_index >= len(gripper_values): + break + + # Check for translation change using the detected position fields + has_translation_change = np.any(np.abs(delta_position_values[base_index]) > EPSILON) + has_gripper_change = gripper_values[base_index] != (previous_gripper if base_index == 0 else gripper_values[base_index-1]) + + if has_translation_change or has_gripper_change: + all_steps.append((trajectory_id, base_index)) + else: + for base_index in range(trajectory_length): + all_steps.append((trajectory_id, base_index)) + + # Print summary statistics + print(f"Single-process summary: Processed {processed_trajectories} trajectories, skipped {skipped_trajectories} empty trajectories") + print(f"Total steps: {len(all_steps)} from {len(self.trajectory_ids)} trajectories") + + return all_steps + + def _get_position_and_gripper_values(self, data: pd.DataFrame) -> tuple[list, list]: + """Get position and gripper values based on available columns in the dataset.""" + # Get action keys from modality_keys + action_keys = self.modality_keys.get('action', []) + + # Extract position data + delta_position_values = None + position_candidates = ['delta_eef_position'] + coordinate_candidates = ['x', 'y', 'z'] + + # First try combined position fields + for pos_key in position_candidates: + full_key = f"action.{pos_key}" + if full_key in action_keys: + try: + # Get the lerobot key for this modality + le_action_cfg = self.lerobot_modality_meta.action + subkey = pos_key + if subkey in le_action_cfg: + le_key = le_action_cfg[subkey].original_key or subkey + if le_key in data.columns: + data_array = np.stack(data[le_key]) + le_indices = np.arange(le_action_cfg[subkey].start, le_action_cfg[subkey].end) + filtered_data = data_array[:, le_indices] + delta_position_values = filtered_data.tolist() + break + except Exception: + continue + + # If combined fields not found, try individual x,y,z coordinates + if delta_position_values is None: + x_data, y_data, z_data = None, None, None + for coord in coordinate_candidates: + full_key = f"action.{coord}" + if full_key in action_keys: + try: + le_action_cfg = self.lerobot_modality_meta.action + if coord in le_action_cfg: + le_key = le_action_cfg[coord].original_key or coord + if le_key in data.columns: + data_array = np.stack(data[le_key]) + le_indices = np.arange(le_action_cfg[coord].start, le_action_cfg[coord].end) + coord_data = data_array[:, le_indices].flatten() + if coord == 'x': + x_data = coord_data + elif coord == 'y': + y_data = coord_data + elif coord == 'z': + z_data = coord_data + except Exception: + continue + + if x_data is not None and y_data is not None and z_data is not None: + delta_position_values = np.column_stack((x_data, y_data, z_data)).tolist() + + if delta_position_values is None: + # Fallback to the old hardcoded approach if metadata approach fails + if 'action.delta_eef_position' in data.columns: + delta_position_values = data['action.delta_eef_position'].to_numpy().tolist() + elif all(col in data.columns for col in ['action.x', 'action.y', 'action.z']): + x_vals = data['action.x'].to_numpy() + y_vals = data['action.y'].to_numpy() + z_vals = data['action.z'].to_numpy() + delta_position_values = np.column_stack((x_vals, y_vals, z_vals)).tolist() + else: + raise ValueError(f"No suitable position columns found. Available columns: {data.columns.tolist()}") + + # Extract gripper data + gripper_values = None + gripper_candidates = ['gripper_close', 'gripper'] + + for grip_key in gripper_candidates: + full_key = f"action.{grip_key}" + if full_key in action_keys: + try: + le_action_cfg = self.lerobot_modality_meta.action + if grip_key in le_action_cfg: + le_key = le_action_cfg[grip_key].original_key or grip_key + if le_key in data.columns: + data_array = np.stack(data[le_key]) + le_indices = np.arange(le_action_cfg[grip_key].start, le_action_cfg[grip_key].end) + gripper_data = data_array[:, le_indices].flatten() + gripper_values = gripper_data.tolist() + break + except Exception: + continue + + if gripper_values is None: + # Fallback to the old hardcoded approach if metadata approach fails + if 'action.gripper_close' in data.columns: + gripper_values = data['action.gripper_close'].to_numpy().tolist() + elif 'action.gripper' in data.columns: + gripper_values = data['action.gripper'].to_numpy().tolist() + else: + raise ValueError(f"No suitable gripper columns found. Available columns: {data.columns.tolist()}") + + return delta_position_values, gripper_values + + def _get_modality_keys(self) -> dict: + """Get the modality keys for the dataset. + The keys are the modality names, and the values are the keys for each modality. + See property `modality_keys` for the expected format. + """ + modality_keys = defaultdict(list) + for modality, config in self.modality_configs.items(): + modality_keys[modality] = config.modality_keys + return modality_keys + + def _get_delta_indices(self) -> dict[str, np.ndarray]: + """Restructure the delta indices to use modality.key as keys instead of just the modalities.""" + delta_indices: dict[str, np.ndarray] = {} + for config in self.modality_configs.values(): + for key in config.modality_keys: + delta_indices[key] = np.array(config.delta_indices) + return delta_indices + + def _get_lerobot_modality_meta(self) -> LeRobotModalityMetadata: + """Get the metadata for the LeRobot dataset.""" + modality_meta_path = self.dataset_path / LE_ROBOT_MODALITY_FILENAME + assert ( + modality_meta_path.exists() + ), f"Please provide a {LE_ROBOT_MODALITY_FILENAME} file in {self.dataset_path}" + with open(modality_meta_path, "r") as f: + modality_meta = LeRobotModalityMetadata.model_validate(json.load(f)) + return modality_meta + + def _get_lerobot_info_meta(self) -> dict: + """Get the metadata for the LeRobot dataset.""" + info_meta_path = self.dataset_path / LE_ROBOT_INFO_FILENAME + with open(info_meta_path, "r") as f: + info_meta = json.load(f) + return info_meta + + def _get_data_path_pattern(self) -> str: + """Get the data path pattern for the LeRobot dataset.""" + return self.lerobot_info_meta["data_path"] + + def _get_video_path_pattern(self) -> str: + """Get the video path pattern for the LeRobot dataset.""" + return self.lerobot_info_meta["video_path"] + + def _get_chunk_size(self) -> int: + """Get the chunk size for the LeRobot dataset.""" + return self.lerobot_info_meta["chunks_size"] + + def _get_tasks(self) -> pd.DataFrame: + """Get the tasks for the dataset.""" + tasks_path = self.dataset_path / LE_ROBOT_TASKS_FILENAME + with open(tasks_path, "r") as f: + tasks = [json.loads(line) for line in f] + df = pd.DataFrame(tasks) + return df.set_index("task_index") + + def _check_integrity(self): + """Use the config to check if the keys are valid and detect silent data corruption.""" + ERROR_MSG_HEADER = f"Error occurred in initializing dataset {self.dataset_name}:\n" + + for modality_config in self.modality_configs.values(): + for key in modality_config.modality_keys: + if key == "lapa_action" or key == "dream_actions": + continue # no need for any metadata for lapa actions because it comes normalized + # Check if the key is valid + try: + self.lerobot_modality_meta.get_key_meta(key) + except Exception as e: + raise ValueError( + ERROR_MSG_HEADER + f"Unable to find key {key} in modality metadata:\n{e}" + ) + + def set_transforms_metadata(self, metadata: DatasetMetadata): + """Set the metadata for the transforms. This is useful for transforms that need to know the metadata, such as the normalization values.""" + self.transforms.set_metadata(metadata) + + def set_epoch(self, epoch: int): + """Set the epoch for the dataset. + + Args: + epoch (int): The epoch to set. + """ + self.epoch = epoch + + def __len__(self) -> int: + """Get the total number of data points in the dataset. + + Returns: + int: the total number of data points in the dataset. + """ + return len(self.all_steps) + + def __str__(self) -> str: + """Get the description of the dataset.""" + return f"{self.dataset_name} ({len(self)} steps)" + + + def __getitem__(self, index: int) -> dict: + """Get the data for a single step in a trajectory. + + Args: + index (int): The index of the step to get. + + Returns: + dict: The data for the step. + """ + trajectory_id, base_index = self.all_steps[index] + data = self.get_step_data(trajectory_id, base_index) + + # Process all video keys dynamically + images = [] + for video_key in self.modality_keys["video"]: + image = data[video_key][0] + + # Apply image cropping if enabled and the video key is base_view + # Note: crop_obs_camera functionality has been removed + + image = Image.fromarray(image).resize((224, 224)) + images.append(image) + + # Get language and action data + language = data[self.modality_keys["language"][0]][0] + action = [] + for action_key in self.modality_keys["action"]: + action.append(data[action_key]) + action = np.concatenate(action, axis=1) + + return dict(action=action, image=images, language=language) + + def get_step_data(self, trajectory_id: int, base_index: int) -> dict: + """Get the RAW data for a single step in a trajectory. No transforms are applied. + + Args: + trajectory_id (int): The name of the trajectory. + base_index (int): The base step index in the trajectory. + + Returns: + dict: The RAW data for the step. + + Example return: + { + "video": { + "video.image_side_0": [B, T, H, W, C], + "video.image_side_1": [B, T, H, W, C], + }, + "state": { + "state.eef_position": [B, T, state_dim], + "state.eef_rotation": [B, T, state_dim], + }, + "action": { + "action.eef_position": [B, T, action_dim], + "action.eef_rotation": [B, T, action_dim], + }, + } + """ + data = {} + # Get the data for all modalities + self.curr_traj_data = self.get_trajectory_data(trajectory_id) + + for modality in self.modality_keys: + # Get the data corresponding to each key in the modality + for key in self.modality_keys[modality]: + data[key] = self.get_data_by_modality(trajectory_id, modality, key, base_index) + return data + + def _preload_all_trajectories(self): + """预加载所有trajectories到内存""" + print("Preloading all trajectories...") + self._trajectory_cache = {} + for trajectory_id in tqdm(self.trajectory_ids, desc="Loading trajectories"): + chunk_index = self.get_episode_chunk(trajectory_id) + parquet_path = self.dataset_path / self.data_path_pattern.format( + episode_chunk=chunk_index, episode_index=trajectory_id + ) + self._trajectory_cache[trajectory_id] = pd.read_parquet(parquet_path) + print(f"Preloaded {len(self._trajectory_cache)} trajectories") + + def get_trajectory_data(self, trajectory_id: int) -> pd.DataFrame: + """Get the data for a trajectory.""" + + if self.preload_all: + return self._trajectory_cache[trajectory_id] + else: + if self.curr_traj_id == trajectory_id and self.curr_traj_data is not None: + return self.curr_traj_data + else: + chunk_index = self.get_episode_chunk(trajectory_id) + parquet_path = self.dataset_path / self.data_path_pattern.format( + episode_chunk=chunk_index, episode_index=trajectory_id + ) + assert parquet_path.exists(), f"Parquet file not found at {parquet_path}" + return pd.read_parquet(parquet_path) + + def get_trajectory_index(self, trajectory_id: int) -> int: + """Get the index of the trajectory in the dataset by the trajectory ID. + This is useful when you need to get the trajectory length or sampling weight corresponding to the trajectory ID. + + Args: + trajectory_id (str): The ID of the trajectory. + + Returns: + int: The index of the trajectory in the dataset. + """ + trajectory_indices = np.where(self.trajectory_ids == trajectory_id)[0] + if len(trajectory_indices) != 1: + raise ValueError( + f"Error finding trajectory index for {trajectory_id}, found {trajectory_indices}" + ) + return trajectory_indices[0] + + def get_episode_chunk(self, ep_index: int) -> int: + """Get the chunk index for an episode index.""" + return ep_index // self.chunk_size + + def retrieve_data_and_pad( + self, + array: np.ndarray, + step_indices: np.ndarray, + max_length: int, + padding_strategy: str = "first_last", + ) -> np.ndarray: + """Retrieve the data from the dataset and pad it if necessary. + Args: + array (np.ndarray): The array to retrieve the data from. + step_indices (np.ndarray): The step indices to retrieve the data for. + max_length (int): The maximum length of the data. + padding_strategy (str): The padding strategy, either "first" or "last". + """ + # Get the padding indices + front_padding_indices = step_indices < 0 + end_padding_indices = step_indices >= max_length + padding_positions = np.logical_or(front_padding_indices, end_padding_indices) + # Retrieve the data with the non-padding indices + # If there exists some padding, Given T step_indices, the shape of the retrieved data will be (T', ...) where T' < T + raw_data = array[step_indices[~padding_positions]] + assert isinstance(raw_data, np.ndarray), f"{type(raw_data)=}" + # This is the shape of the output, (T, ...) + if raw_data.ndim == 1: + expected_shape = (len(step_indices),) + else: + expected_shape = (len(step_indices), *array.shape[1:]) + + # Pad the data + output = np.zeros(expected_shape) + # Assign the non-padded data + output[~padding_positions] = raw_data + # If there exists some padding, pad the data + if padding_positions.any(): + if padding_strategy == "first_last": + # Use first / last step data to pad + front_padding_data = array[0] + end_padding_data = array[-1] + output[front_padding_indices] = front_padding_data + output[end_padding_indices] = end_padding_data + elif padding_strategy == "zero": + # Use zero padding + output[padding_positions] = 0 + else: + raise ValueError(f"Invalid padding strategy: {padding_strategy}") + return output + + def get_video_path(self, trajectory_id: int, key: str) -> Path: + chunk_index = self.get_episode_chunk(trajectory_id) + original_key = self.lerobot_modality_meta.video[key].original_key + if original_key is None: + original_key = key + video_filename = self.video_path_pattern.format( + episode_chunk=chunk_index, episode_index=trajectory_id, video_key=original_key + ) + return self.dataset_path / video_filename + + def get_video( + self, + trajectory_id: int, + key: str, + base_index: int, + ) -> np.ndarray: + """Get the video frames for a trajectory by a base index. + + Args: + dataset (BaseSingleDataset): The dataset to retrieve the data from. + trajectory_id (str): The ID of the trajectory. + key (str): The key of the video. + base_index (int): The base index of the trajectory. + + Returns: + np.ndarray: The video frames for the trajectory and frame indices. Shape: (T, H, W, C) + """ + # Get the step indices + step_indices = self.delta_indices[key] + base_index + # print(f"{step_indices=}") + # Get the trajectory index + trajectory_index = self.get_trajectory_index(trajectory_id) + # Ensure the indices are within the valid range + # This is equivalent to padding the video with extra frames at the beginning and end + step_indices = np.maximum(step_indices, 0) + step_indices = np.minimum(step_indices, self.trajectory_lengths[trajectory_index] - 1) + assert key.startswith("video."), f"Video key must start with 'video.', got {key}" + # Get the sub-key + key = key.replace("video.", "") + video_path = self.get_video_path(trajectory_id, key) + # Get the action/state timestamps for each frame in the video + assert self.curr_traj_data is not None, f"No data found for {trajectory_id}" + assert "timestamp" in self.curr_traj_data.columns, f"No timestamp found in {trajectory_id}" + timestamp: np.ndarray = self.curr_traj_data["timestamp"].to_numpy() + # Get the corresponding video timestamps from the step indices + video_timestamp = timestamp[step_indices] + + return get_frames_by_timestamps( + video_path.as_posix(), + video_timestamp, + video_backend=self.video_backend, + video_backend_kwargs=self.video_backend_kwargs, + step_indices_for_assert=step_indices, + ) + + def get_state_or_action( + self, + trajectory_id: int, + modality: str, + key: str, + base_index: int, + ) -> np.ndarray: + """Get the state or action data for a trajectory by a base index. + If the step indices are out of range, pad with the data: + if the data is stored in absolute format, pad with the first or last step data; + otherwise, pad with zero. + + Args: + dataset (BaseSingleDataset): The dataset to retrieve the data from. + trajectory_id (int): The ID of the trajectory. + modality (str): The modality of the data. + key (str): The key of the data. + base_index (int): The base index of the trajectory. + + Returns: + np.ndarray: The data for the trajectory and step indices. + """ + # Get the step indices + step_indices = self.delta_indices[key] + base_index + # Get the trajectory index + trajectory_index = self.get_trajectory_index(trajectory_id) + # Get the maximum length of the trajectory + max_length = self.trajectory_lengths[trajectory_index] + assert key.startswith(modality + "."), f"{key} must start with {modality + '.'}, got {key}" + # Get the sub-key, e.g. state.joint_angles -> joint_angles + key = key.replace(modality + ".", "") + # Get the lerobot key + le_state_or_action_cfg = getattr(self.lerobot_modality_meta, modality) + le_key = le_state_or_action_cfg[key].original_key + if le_key is None: + le_key = key + # Get the data array, shape: (T, D) + assert self.curr_traj_data is not None, f"No data found for {trajectory_id}" + assert le_key in self.curr_traj_data.columns, f"No {le_key} found in {trajectory_id}" + data_array: np.ndarray = np.stack(self.curr_traj_data[le_key]) # type: ignore + assert data_array.ndim == 2, f"Expected 2D array, got key {le_key} is{data_array.shape} array" + le_indices = np.arange( + le_state_or_action_cfg[key].start, + le_state_or_action_cfg[key].end, + ) + data_array = data_array[:, le_indices] + # Get the state or action configuration + state_or_action_cfg = getattr(self.metadata.modalities, modality)[key] + + # Pad the data + return self.retrieve_data_and_pad( + array=data_array, + step_indices=step_indices, + max_length=max_length, + padding_strategy="first_last" if state_or_action_cfg.absolute else "zero", + # padding_strategy="first_last", # HACK for realdata + ) + + def get_language( + self, + trajectory_id: int, + key: str, + base_index: int, + ) -> list[str]: + """Get the language annotation data for a trajectory by step indices. + + Args: + dataset (BaseSingleDataset): The dataset to retrieve the data from. + trajectory_id (int): The ID of the trajectory. + key (str): The key of the annotation. + base_index (int): The base index of the trajectory. + + Returns: + list[str]: The annotation data for the trajectory and step indices. If no matching data is found, return empty strings. + """ + assert self.curr_traj_data is not None, f"No data found for {trajectory_id}" + # Get the step indices + step_indices = self.delta_indices[key] + base_index + # Get the trajectory index + trajectory_index = self.get_trajectory_index(trajectory_id) + # Get the maximum length of the trajectory + max_length = self.trajectory_lengths[trajectory_index] + # Get the end times corresponding to the closest indices + step_indices = np.maximum(step_indices, 0) + step_indices = np.minimum(step_indices, max_length - 1) + # Get the annotations + task_indices: list[int] = [] + assert key.startswith( + "annotation." + ), f"Language key must start with 'annotation.', got {key}" + subkey = key.replace("annotation.", "") + annotation_meta = self.lerobot_modality_meta.annotation + assert annotation_meta is not None, f"Annotation metadata is None for {subkey}" + assert ( + subkey in annotation_meta + ), f"Annotation key {subkey} not found in metadata, available annotation keys: {annotation_meta.keys()}" + subkey_meta = annotation_meta[subkey] + original_key = subkey_meta.original_key + if original_key is None: + original_key = key + for i in range(len(step_indices)): + task_indices.append(self.curr_traj_data[original_key][step_indices[i]].item()) + return self.tasks.loc[task_indices]["description"].tolist() # 适配zhenyu脚本 + + def get_data_by_modality( + self, + trajectory_id: int, + modality: str, + key: str, + base_index: int, + ): + """Get the data corresponding to the modality for a trajectory by a base index. + This method will call the corresponding helper method based on the modality. + See the helper methods for more details. + NOTE: For the language modality, the data is padded with empty strings if no matching data is found. + + Args: + dataset (BaseSingleDataset): The dataset to retrieve the data from. + trajectory_id (int): The ID of the trajectory. + modality (str): The modality of the data. + key (str): The key of the data. + base_index (int): The base index of the trajectory. + """ + if modality == "video": + return self.get_video(trajectory_id, key, base_index) + elif modality == "state" or modality == "action": + return self.get_state_or_action(trajectory_id, modality, key, base_index) + elif modality == "language": + return self.get_language(trajectory_id, key, base_index) + else: + raise ValueError(f"Invalid modality: {modality}") + + def save_dataset_statistics(self, save_path: Path | str, format: str = "json") -> None: + """ + Save dataset statistics to specified path in the required format. + Only includes statistics for keys that are actually used in the dataset. + Gripper-related keys will be placed at the end. + + Args: + save_path (Path | str): Path to save the statistics file + format (str): Save format, currently only supports "json" + """ + save_path = Path(save_path) + save_path.parent.mkdir(parents=True, exist_ok=True) + + # Build the data structure to save + statistics_data = {} + + # Get used modality keys + used_action_keys, used_state_keys = get_used_modality_keys(self.modality_keys) + + # Organize statistics by tag + tag = self.tag + tag_stats = {} + + # Process action statistics (only for used keys) + if hasattr(self.metadata.statistics, 'action') and self.metadata.statistics.action: + action_stats = self.metadata.statistics.action + + # Filter to only include used action keys and reorder: non-gripper first, gripper last + non_gripper_keys = [] + gripper_keys = [] + + for key in action_stats.keys(): + if key in used_action_keys: + if "gripper" in key.lower(): + gripper_keys.append(key) + else: + non_gripper_keys.append(key) + + # Reorder: non-gripper first, gripper last + reordered_keys = non_gripper_keys + gripper_keys + + filtered_action_stats = {} + for key in reordered_keys: + filtered_action_stats[key] = action_stats[key] + + if filtered_action_stats: + # Combine statistics from filtered action sub-keys + combined_action_stats = combine_modality_stats(filtered_action_stats) + + # Add mask field: based on whether it's gripper or not + mask = generate_action_mask_for_used_keys( + self.metadata.modalities.action, filtered_action_stats.keys() + ) + combined_action_stats["mask"] = mask + + tag_stats["action"] = combined_action_stats + + # Process state statistics (only for used keys) + if hasattr(self.metadata.statistics, 'state') and self.metadata.statistics.state: + state_stats = self.metadata.statistics.state + + # Filter to only include used state keys, optionally reorder gripper to end + non_gripper_keys = [] + gripper_keys = [] + + for key in state_stats.keys(): + if key in used_state_keys: + if "gripper" in key.lower(): + gripper_keys.append(key) + else: + non_gripper_keys.append(key) + + # Reorder: non-gripper first, gripper last + reordered_keys = non_gripper_keys + gripper_keys + + filtered_state_stats = {} + for key in reordered_keys: + filtered_state_stats[key] = state_stats[key] + + if filtered_state_stats: + combined_state_stats = combine_modality_stats(filtered_state_stats) + tag_stats["state"] = combined_state_stats + + # Add dataset counts + tag_stats["num_transitions"] = len(self) + tag_stats["num_trajectories"] = len(self.trajectory_ids) + + statistics_data[tag] = tag_stats + + # Save as JSON file + if format.lower() == "json": + if not str(save_path).endswith('.json'): + save_path = save_path.with_suffix('.json') + with open(save_path, 'w', encoding='utf-8') as f: + json.dump(statistics_data, f, indent=2, ensure_ascii=False) + else: + raise ValueError(f"Unsupported format: {format}. Currently only 'json' is supported.") + + print(f"Single dataset statistics saved to: {save_path}") + print(f"Used action keys (reordered): {list(used_action_keys)}") + print(f"Used state keys (reordered): {list(used_state_keys)}") + + +class CachedLeRobotSingleDataset(LeRobotSingleDataset): + def __init__(self, img_resize: tuple[int, int] | None = None, *args, **kwargs): + """ + This class caches the video frames for each trajectory and key. + It is recommended to use this class if the video frames need to be accessed multiple times. + + Args: + resize_img (tuple[int, int], optional): The size to resize the video frames to reduce memory usage. + """ + # Convert img_resize to tuple if it is not already + if img_resize is not None and not isinstance(img_resize, tuple): + img_resize = tuple(img_resize) + assert len(img_resize) == 2, f"Expected tuple of length 2, got {img_resize}" + self.img_resize = img_resize + + # Initialize img_resize attribute first to ensure it exists + super().__init__(*args, **kwargs) + cached_frames: dict[str, np.ndarray] = {} + + for key in self.modality_keys["video"]: + all_frames = [] + original_key = key + key = key.replace("video.", "") + for trajectory_id, trajectory_length in tqdm( + zip(self.trajectory_ids, self.trajectory_lengths), + total=len(self.trajectory_ids), + desc=f"Caching {key} frames", + ): + video_path = self.get_video_path(trajectory_id, key) + frames = get_all_frames( + video_path.as_posix(), + video_backend=self.video_backend, + video_backend_kwargs=self.video_backend_kwargs, + resize_size=img_resize, + ) + assert frames.ndim == 4, f"Expected 4D array, got {frames.shape} array" + assert frames.shape[3] == 3, f"Expected 3 channels, got {frames.shape[3]} channels" + + # Apply image cropping if enabled and the video key is base_view + # Note: crop_obs_camera functionality has been removed + + # assert ( + # frames.shape[0] == trajectory_length + # ), f"Expected {trajectory_length} frames, got {frames.shape[0]} frames" + all_frames.append(frames) + cached_frames[key] = np.concatenate(all_frames, axis=0) + print(f"{key}: {cached_frames[key].shape}") + self.cached_frames = cached_frames + self.start_indices = np.cumsum(self.trajectory_lengths) - self.trajectory_lengths + + def get_video(self, trajectory_id: int, key: str, base_index: int) -> np.ndarray: + step_indices = self.delta_indices[key] + base_index + # Get the trajectory index + trajectory_index = self.get_trajectory_index(trajectory_id) + # Ensure the indices are within the valid range + # This is equivalent to padding the video with extra frames at the beginning and end + step_indices = np.maximum(step_indices, 0) + step_indices = np.minimum(step_indices, self.trajectory_lengths[trajectory_index] - 1) + assert key.startswith("video."), f"Video key must start with 'video.', got {key}" + # Get the sub-key + key = key.replace("video.", "") + # Calculate the absolute indices + absolute_indices = self.start_indices[trajectory_index] + step_indices + return self.cached_frames[key][absolute_indices] + + def get_step_data(self, trajectory_id: int, base_index: int) -> dict: + """Get the RAW data for a single step. No transforms are applied. + + Args: + trajectory_id (str): The ID of the trajectory. + base_index (int): The base index of the step. + + Returns: + dict: The data for the step. + """ + data = {} + self.curr_traj_data = self.get_trajectory_data(trajectory_id) + # Get the data for all modalities + for modality in self.modality_keys: + # Get the data corresponding to each key in the modality + for key in self.modality_keys[modality]: + data[key] = self.get_data_by_modality(trajectory_id, modality, key, base_index) + return data + + def set_transforms_metadata(self, metadata: DatasetMetadata): + """Set the metadata for the transforms. This is useful for transforms that need to know the metadata, such as the normalization values.""" + if self.img_resize is not None: + all_video_keys = [key for key in self.modality_keys["video"]] + for key in metadata.modalities.video: + if key in all_video_keys: + metadata.modalities.video[key].resolution = self.img_resize + super().set_transforms_metadata(metadata) + + +def safe_hash(input_tuple): + # keep 128 bits of the hash + tuple_string = repr(input_tuple).encode("utf-8") + sha256 = hashlib.sha256() + sha256.update(tuple_string) + + seed = int(sha256.hexdigest(), 16) + + return seed & 0xFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF + + +class MixtureSpecElement(BaseModel): + dataset_path: list[Path] | Path = Field(..., description="The path to the dataset.") + dataset_weight: float = Field(..., description="The weight of the dataset in the mixture.") + distribute_weights: bool = Field( + default=False, + description="Whether to distribute the weights of the dataset across all the paths. If True, the weights will be evenly distributed across all the paths.", + ) + + +# Helper functions for dataset statistics + +def combine_modality_stats(modality_stats: dict) -> dict: + """ + Combine statistics from all sub-keys under a modality. + + Args: + modality_stats (dict): Statistics for a modality, containing multiple sub-keys. + Each sub-key contains DatasetStatisticalValues object. + + Returns: + dict: Combined statistics + """ + combined_stats = { + "mean": [], + "std": [], + "max": [], + "min": [], + "q01": [], + "q99": [] + } + + # Combine statistics in sub-key order + for subkey in modality_stats.keys(): + subkey_stats = modality_stats[subkey] # This is a DatasetStatisticalValues object + + # Convert DatasetStatisticalValues to dict-like access + for stat_name in ["mean", "std", "max", "min", "q01", "q99"]: + stat_value = getattr(subkey_stats, stat_name) + if isinstance(stat_value, (list, tuple)): + combined_stats[stat_name].extend(stat_value) + else: + # Handle NDArray case - convert to list + if hasattr(stat_value, 'tolist'): + combined_stats[stat_name].extend(stat_value.tolist()) + else: + combined_stats[stat_name].append(float(stat_value)) + + return combined_stats + +def generate_action_mask_for_used_keys(action_modalities: dict, used_action_keys_ordered) -> list[bool]: + """ + Generate mask based on action modalities, but only for used keys. + Gripper-related are False, others are True. + + Args: + action_modalities (dict): Configuration information for action modalities. + used_action_keys_ordered: Iterable of actually used action keys in the correct order. + + Returns: + list[bool]: List of mask values + """ + mask = [] + + # Generate mask in the same order as the statistics were combined + for subkey in used_action_keys_ordered: + if subkey in action_modalities: + subkey_config = action_modalities[subkey] + + # Get dimension count from shape + if hasattr(subkey_config, 'shape') and len(subkey_config.shape) > 0: + dim_count = subkey_config.shape[0] + else: + dim_count = 1 + + # Check if it's gripper-related + is_gripper = "gripper" in subkey.lower() + + # Generate mask value for each dimension + for _ in range(dim_count): + mask.append(not is_gripper) # gripper is False, others are True + + return mask + +def get_used_modality_keys(modality_keys: dict) -> tuple[set, set]: + """Extract used action and state keys from modality configuration.""" + used_action_keys = set() + used_state_keys = set() + + # Extract action keys (remove "action." prefix) + for action_key in modality_keys.get("action", []): + if action_key.startswith("action."): + clean_key = action_key.replace("action.", "") + used_action_keys.add(clean_key) + + # Extract state keys (remove "state." prefix) + for state_key in modality_keys.get("state", []): + if state_key.startswith("state."): + clean_key = state_key.replace("state.", "") + used_state_keys.add(clean_key) + + return used_action_keys, used_state_keys + +class LeRobotMixtureDataset(Dataset): + """ + A mixture of multiple datasets. This class samples a single dataset based on the dataset weights and then calls the `__getitem__` method of the sampled dataset. + It is recommended to modify the single dataset class instead of this class. + """ + + def __init__( + self, + data_mixture: Sequence[tuple[LeRobotSingleDataset, float]], + mode: str, + balance_dataset_weights: bool = True, + balance_trajectory_weights: bool = True, + seed: int = 42, + metadata_config: dict = { + "percentile_mixing_method": "min_max", + }, + ): + """ + Initialize the mixture dataset. + + Args: + data_mixture (list[tuple[LeRobotSingleDataset, float]]): Datasets and their corresponding weights. + mode (str): If "train", __getitem__ will return different samples every epoch; if "val" or "test", __getitem__ will return the same sample every epoch. + balance_dataset_weights (bool): If True, the weight of dataset will be multiplied by the total trajectory length of each dataset. + balance_trajectory_weights (bool): If True, sample trajectories within a dataset weighted by their length; otherwise, use equal weighting. + seed (int): Random seed for sampling. + """ + datasets: list[LeRobotSingleDataset] = [] + dataset_sampling_weights: list[float] = [] + for dataset, weight in data_mixture: + # Check if dataset is valid and has data + if len(dataset) == 0: + print(f"Warning: Skipping empty dataset {dataset.dataset_name}") + continue + datasets.append(dataset) + dataset_sampling_weights.append(weight) + + if len(datasets) == 0: + raise ValueError("No valid datasets found in the mixture. All datasets are empty.") + + self.datasets = datasets + self.balance_dataset_weights = balance_dataset_weights + self.balance_trajectory_weights = balance_trajectory_weights + self.seed = seed + self.mode = mode + + # Set properties for sampling + + # 1. Dataset lengths + self._dataset_lengths = np.array([len(dataset) for dataset in self.datasets]) + print(f"Dataset lengths: {self._dataset_lengths}") + + # 2. Dataset sampling weights + self._dataset_sampling_weights = np.array(dataset_sampling_weights) + + if self.balance_dataset_weights: + self._dataset_sampling_weights *= self._dataset_lengths + + # Check for zero or negative weights before normalization + if np.any(self._dataset_sampling_weights <= 0): + print(f"Warning: Found zero or negative sampling weights: {self._dataset_sampling_weights}") + # Set minimum weight to prevent division issues + self._dataset_sampling_weights = np.maximum(self._dataset_sampling_weights, 1e-8) + + # Normalize weights + weights_sum = self._dataset_sampling_weights.sum() + if weights_sum == 0 or np.isnan(weights_sum): + print(f"Error: Invalid weights sum: {weights_sum}") + # Fallback to equal weights + self._dataset_sampling_weights = np.ones(len(self.datasets)) / len(self.datasets) + print(f"Fallback to equal weights") + else: + self._dataset_sampling_weights /= weights_sum + + # 3. Trajectory sampling weights + self._trajectory_sampling_weights: list[np.ndarray] = [] + for i, dataset in enumerate(self.datasets): + trajectory_sampling_weights = np.ones(len(dataset.trajectory_lengths)) + if self.balance_trajectory_weights: + trajectory_sampling_weights *= dataset.trajectory_lengths + + # Check for zero or negative weights before normalization + if np.any(trajectory_sampling_weights <= 0): + print(f"Warning: Dataset {i} has zero or negative trajectory weights") + trajectory_sampling_weights = np.maximum(trajectory_sampling_weights, 1e-8) + + # Normalize weights + weights_sum = trajectory_sampling_weights.sum() + if weights_sum == 0 or np.isnan(weights_sum): + print(f"Error: Dataset {i} has invalid trajectory weights sum: {weights_sum}") + # Fallback to equal weights + trajectory_sampling_weights = np.ones(len(dataset.trajectory_lengths)) / len(dataset.trajectory_lengths) + else: + trajectory_sampling_weights /= weights_sum + + self._trajectory_sampling_weights.append(trajectory_sampling_weights) + + # 4. Primary dataset indices + self._primary_dataset_indices = np.array(dataset_sampling_weights) == 1.0 + if not np.any(self._primary_dataset_indices): + print(f"Warning: No dataset with weight 1.0 found. Original weights: {dataset_sampling_weights}") + # Fallback: use the dataset(s) with maximum weight as primary + max_weight = max(dataset_sampling_weights) + self._primary_dataset_indices = np.array(dataset_sampling_weights) == max_weight + print(f"Using datasets with maximum weight {max_weight} as primary: {self._primary_dataset_indices}") + + if not np.any(self._primary_dataset_indices): + # This should never happen, but just in case + print("Error: Still no primary dataset found. Using first dataset as primary.") + self._primary_dataset_indices = np.zeros(len(self.datasets), dtype=bool) + self._primary_dataset_indices[0] = True + + # Set the epoch and sample the first epoch + self.set_epoch(0) + + self.update_metadata(metadata_config) + + @property + def dataset_lengths(self) -> np.ndarray: + """The lengths of each dataset.""" + return self._dataset_lengths + + @property + def dataset_sampling_weights(self) -> np.ndarray: + """The sampling weights for each dataset.""" + return self._dataset_sampling_weights + + @property + def trajectory_sampling_weights(self) -> list[np.ndarray]: + """The sampling weights for each trajectory in each dataset.""" + return self._trajectory_sampling_weights + + @property + def primary_dataset_indices(self) -> np.ndarray: + """The indices of the primary datasets.""" + return self._primary_dataset_indices + + def __str__(self) -> str: + dataset_descriptions = [] + for dataset, weight in zip(self.datasets, self.dataset_sampling_weights): + dataset_description = { + "Dataset": str(dataset), + "Sampling weight": float(weight), + } + dataset_descriptions.append(dataset_description) + return json.dumps({"Mixture dataset": dataset_descriptions}, indent=2) + + def set_epoch(self, epoch: int): + """Set the epoch for the dataset. + + Args: + epoch (int): The epoch to set. + """ + self.epoch = epoch + # self.sampled_steps = self.sample_epoch() + + def sample_step(self, index: int) -> tuple[LeRobotSingleDataset, int, int]: + """Sample a single step from the dataset.""" + # return self.sampled_steps[index] + + # Set seed + seed = index if self.mode != "train" else safe_hash((self.epoch, index, self.seed)) + rng = np.random.default_rng(seed) + + # Sample dataset + dataset_index = rng.choice(len(self.datasets), p=self.dataset_sampling_weights) + dataset = self.datasets[dataset_index] + + # Sample trajectory + # trajectory_index = rng.choice( + # len(dataset.trajectory_ids), p=self.trajectory_sampling_weights[dataset_index] + # ) + # trajectory_id = dataset.trajectory_ids[trajectory_index] + + # # Sample step + # base_index = rng.choice(dataset.trajectory_lengths[trajectory_index]) + # return dataset, trajectory_id, base_index + single_step_index = rng.choice(len(dataset.all_steps)) + trajectory_id, base_index = dataset.all_steps[single_step_index] + return dataset, trajectory_id, base_index + + def sample_step_with_shuffle_is_true(self, index: int) -> tuple[LeRobotSingleDataset, int, int]: + """Sample a single step from the dataset.""" + '''only use this when DataLoader is initialized with shuffle=True''' + print(index) + # return self.sampled_steps[index] + + # Set seed + # seed = index if self.mode != "train" else safe_hash((self.epoch, index, self.seed)) + # rng = np.random.default_rng(seed) + + # Sample dataset + # dataset_index = rng.choice(len(self.datasets), p=self.dataset_sampling_weights) + # dataset = self.datasets[dataset_index] + dataset = self.datasets[0] #FIXME + + # Sample trajectory + # trajectory_index = rng.choice( + # len(dataset.trajectory_ids), p=self.trajectory_sampling_weights[dataset_index] + # ) + # trajectory_id = dataset.trajectory_ids[trajectory_index] + + # # Sample step + # base_index = rng.choice(dataset.trajectory_lengths[trajectory_index]) + # return dataset, trajectory_id, base_index + # no random, follow the order of the dataset + # single_step_index = rng.choice(len(dataset.all_steps)) + # trajectory_id, base_index = dataset.all_steps[single_step_index] + + trajectory_id, base_index = dataset.all_steps[index] + return dataset, trajectory_id, base_index + + def __getitem__(self, index: int) -> dict: + """Get the data for a single trajectory and start index. + + Args: + index (int): The index of the trajectory to get. + + Returns: + dict: The data for the trajectory and start index. + """ + max_retries = 10 + last_exception = None + + for attempt in range(max_retries): + try: + dataset, trajectory_name, step = self.sample_step(index) + data = dataset.transforms(dataset.get_step_data(trajectory_name, step)) + + # Process all video keys dynamically + images = [] + for video_key in dataset.modality_keys["video"]: + image = data[video_key][0] + + # Apply image cropping if enabled and the video key is base_view + # Note: crop_obs_camera functionality has been removed + + image = Image.fromarray(image).resize((224, 224)) + # # use time stamp as filename + # import time + # filename = f"/hfm/boqian/storage/tmp/image_{int(time.time())}_{index}.png" + # image.save(filename) + images.append(image) + + # Get language and action data + language = data[dataset.modality_keys["language"][0]][0] + action = [] + for action_key in dataset.modality_keys["action"]: + action.append(data[action_key]) + action = np.concatenate(action, axis=1).astype(np.float16) + + return dict(action=action, image=images, lang=language) + + except Exception as e: + last_exception = e + if attempt < max_retries - 1: + # Log the error but continue trying + print(f"Attempt {attempt + 1}/{max_retries} failed for index {index}: {e}") + print(f"Retrying with new sample...") + # For retry, we can use a slightly different index to get a new sample + # This helps avoid getting stuck on the same problematic sample + index = (index + 1) % len(self) + else: + # All retries exhausted + print(f"All {max_retries} attempts failed for index {index}") + print(f"Last error: {last_exception}") + # Return a dummy sample or re-raise the exception + raise last_exception + + def __len__(self) -> int: + """Get the length of a single epoch in the mixture. + + Returns: + int: The length of a single epoch in the mixture. + """ + # Check for potential issues + if len(self.datasets) == 0: + return 0 + + # Check if any dataset lengths are 0 or NaN + if np.any(self.dataset_lengths == 0) or np.any(np.isnan(self.dataset_lengths)): + print(f"Warning: Found zero or NaN dataset lengths: {self.dataset_lengths}") + # Filter out zero/NaN length datasets + valid_indices = (self.dataset_lengths > 0) & (~np.isnan(self.dataset_lengths)) + if not np.any(valid_indices): + print("Error: All datasets have zero or NaN length") + return 0 + else: + valid_indices = np.ones(len(self.datasets), dtype=bool) + + # Check if any sampling weights are 0 or NaN + if np.any(self.dataset_sampling_weights == 0) or np.any(np.isnan(self.dataset_sampling_weights)): + print(f"Warning: Found zero or NaN sampling weights: {self.dataset_sampling_weights}") + # Use only valid weights + valid_weights = (self.dataset_sampling_weights > 0) & (~np.isnan(self.dataset_sampling_weights)) + valid_indices = valid_indices & valid_weights + if not np.any(valid_indices): + print("Error: All sampling weights are zero or NaN") + return 0 + + # Check primary dataset indices + primary_and_valid = self.primary_dataset_indices & valid_indices + if not np.any(primary_and_valid): + print(f"Warning: No valid primary datasets found. Primary indices: {self.primary_dataset_indices}, Valid indices: {valid_indices}") + # Fallback: use the largest valid dataset + if np.any(valid_indices): + max_length = self.dataset_lengths[valid_indices].max() + print(f"Fallback: Using maximum dataset length: {max_length}") + return int(max_length) + else: + return 0 + + # Calculate the ratio and get max + ratios = (self.dataset_lengths / self.dataset_sampling_weights)[primary_and_valid] + + # Check for NaN or inf in ratios + if np.any(np.isnan(ratios)) or np.any(np.isinf(ratios)): + print(f"Warning: Found NaN or inf in ratios: {ratios}") + print(f"Dataset lengths: {self.dataset_lengths[primary_and_valid]}") + print(f"Sampling weights: {self.dataset_sampling_weights[primary_and_valid]}") + # Filter out invalid ratios + valid_ratios = ratios[~np.isnan(ratios) & ~np.isinf(ratios)] + if len(valid_ratios) == 0: + print("Error: All ratios are NaN or inf") + return 0 + max_ratio = valid_ratios.max() + else: + max_ratio = ratios.max() + + result = int(max_ratio) + if result == 0: + print(f"Warning: Dataset mixture length is 0") + return result + + @staticmethod + def compute_overall_statistics( + per_task_stats: list[dict[str, dict[str, list[float] | np.ndarray]]], + dataset_sampling_weights: list[float] | np.ndarray, + percentile_mixing_method: str = "weighted_average", + ) -> dict[str, dict[str, list[float]]]: + """ + Computes overall statistics from per-task statistics using dataset sample weights. + + Args: + per_task_stats: List of per-task statistics. + Example format of one element in the per-task statistics list: + { + "state.gripper": { + "min": [...], + "max": [...], + "mean": [...], + "std": [...], + "q01": [...], + "q99": [...], + }, + ... + } + dataset_sampling_weights: List of sample weights for each task. + percentile_mixing_method: The method to mix the percentiles, either "weighted_average" or "weighted_std". + + Returns: + A dict of overall statistics per modality. + """ + # Normalize the sample weights to sum to 1 + dataset_sampling_weights = np.array(dataset_sampling_weights) + normalized_weights = dataset_sampling_weights / dataset_sampling_weights.sum() + + # Initialize overall statistics dict + overall_stats: dict[str, dict[str, list[float]]] = {} + + # Get the list of modality keys + modality_keys = per_task_stats[0].keys() + + for modality in modality_keys: + # Number of dimensions (assuming consistent across tasks) + num_dims = len(per_task_stats[0][modality]["mean"]) + + # Initialize accumulators for means and variances + weighted_means = np.zeros(num_dims) + weighted_squares = np.zeros(num_dims) + + # Collect min, max, q01, q99 from all tasks + min_list = [] + max_list = [] + q01_list = [] + q99_list = [] + + for task_idx, task_stats in enumerate(per_task_stats): + w_i = normalized_weights[task_idx] + stats = task_stats[modality] + means = np.array(stats["mean"]) + stds = np.array(stats["std"]) + + # Update weighted sums for mean and variance + weighted_means += w_i * means + weighted_squares += w_i * (stds**2 + means**2) + + # Collect min, max, q01, q99 + min_list.append(stats["min"]) + max_list.append(stats["max"]) + q01_list.append(stats["q01"]) + q99_list.append(stats["q99"]) + + # Compute overall mean + overall_mean = weighted_means.tolist() + + # Compute overall variance and std deviation + overall_variance = weighted_squares - weighted_means**2 + overall_std = np.sqrt(overall_variance).tolist() + + # Compute overall min and max per dimension + overall_min = np.min(np.array(min_list), axis=0).tolist() + overall_max = np.max(np.array(max_list), axis=0).tolist() + + # Compute overall q01 and q99 per dimension + # Use weighted average of per-task quantiles + q01_array = np.array(q01_list) + q99_array = np.array(q99_list) + if percentile_mixing_method == "weighted_average": + weighted_q01 = np.average(q01_array, axis=0, weights=normalized_weights).tolist() + weighted_q99 = np.average(q99_array, axis=0, weights=normalized_weights).tolist() + # std_q01 = np.std(q01_array, axis=0).tolist() + # std_q99 = np.std(q99_array, axis=0).tolist() + # print(modality) + # print(f"{std_q01=}, {std_q99=}") + # print(f"{weighted_q01=}, {weighted_q99=}") + elif percentile_mixing_method == "min_max": + weighted_q01 = np.min(q01_array, axis=0).tolist() + weighted_q99 = np.max(q99_array, axis=0).tolist() + else: + raise ValueError(f"Invalid percentile mixing method: {percentile_mixing_method}") + + # Store the overall statistics for the modality + overall_stats[modality] = { + "min": overall_min, + "max": overall_max, + "mean": overall_mean, + "std": overall_std, + "q01": weighted_q01, + "q99": weighted_q99, + } + + return overall_stats + + @staticmethod + def merge_metadata( + metadatas: list[DatasetMetadata], + dataset_sampling_weights: list[float], + percentile_mixing_method: str, + ) -> DatasetMetadata: + """Merge multiple metadata into one.""" + # Convert to dicts + metadata_dicts = [metadata.model_dump(mode="json") for metadata in metadatas] + # Create a new metadata dict + merged_metadata = {} + + # Check all metadata have the same embodiment tag + assert all( + metadata.embodiment_tag == metadatas[0].embodiment_tag for metadata in metadatas + ), "All metadata must have the same embodiment tag" + merged_metadata["embodiment_tag"] = metadatas[0].embodiment_tag + + # Merge the dataset statistics + dataset_statistics = {} + dataset_statistics["state"] = LeRobotMixtureDataset.compute_overall_statistics( + per_task_stats=[m["statistics"]["state"] for m in metadata_dicts], + dataset_sampling_weights=dataset_sampling_weights, + percentile_mixing_method=percentile_mixing_method, + ) + dataset_statistics["action"] = LeRobotMixtureDataset.compute_overall_statistics( + per_task_stats=[m["statistics"]["action"] for m in metadata_dicts], + dataset_sampling_weights=dataset_sampling_weights, + percentile_mixing_method=percentile_mixing_method, + ) + merged_metadata["statistics"] = dataset_statistics + + # Merge the modality configs + modality_configs = defaultdict(set) + for metadata in metadata_dicts: + for modality, configs in metadata["modalities"].items(): + modality_configs[modality].add(json.dumps(configs)) + merged_metadata["modalities"] = {} + for modality, configs in modality_configs.items(): + # Check that all modality configs correspond to the same tag matches + assert ( + len(configs) == 1 + ), f"Multiple modality configs for modality {modality}: {list(configs)}" + merged_metadata["modalities"][modality] = json.loads(configs.pop()) + + return DatasetMetadata.model_validate(merged_metadata) + + def update_metadata(self, metadata_config: dict, cached_statistics_path: Path | str | None = None) -> None: + """ + Merge multiple metadatas into one and set the transforms with the merged metadata. + + Args: + metadata_config (dict): Configuration for the metadata. + "percentile_mixing_method": The method to mix the percentiles, either "weighted_average" or "min_max". + weighted_average: Use the weighted average of the percentiles using the weight used in sampling the datasets. + min_max: Use the min of the 1st percentile and max of the 99th percentile. + """ + # If cached path is provided, try to load and apply + if cached_statistics_path is not None: + try: + cached_stats = self.load_merged_statistics(cached_statistics_path) + self.apply_cached_statistics(cached_stats) + return + except (FileNotFoundError, KeyError, ValidationError) as e: + print(f"Failed to load cached statistics: {e}") + print("Falling back to computing statistics from scratch...") + + self.tag = EmbodimentTag.NEW_EMBODIMENT.value + self.merged_metadata: dict[str, DatasetMetadata] = {} + # Group metadata by tag + all_metadatas: dict[str, list[DatasetMetadata]] = {} + for dataset in self.datasets: + if dataset.tag not in all_metadatas: + all_metadatas[dataset.tag] = [] + all_metadatas[dataset.tag].append(dataset.metadata) + for tag, metadatas in all_metadatas.items(): + self.merged_metadata[tag] = self.merge_metadata( + metadatas=metadatas, + dataset_sampling_weights=self.dataset_sampling_weights.tolist(), + percentile_mixing_method=metadata_config["percentile_mixing_method"], + ) + for dataset in self.datasets: + dataset.set_transforms_metadata(self.merged_metadata[dataset.tag]) + + def save_dataset_statistics(self, save_path: Path | str, format: str = "json") -> None: + """ + Save merged dataset statistics to specified path in the required format. + Only includes statistics for keys that are actually used in the datasets. + Gripper-related keys will be placed at the end. + + Args: + save_path (Path | str): Path to save the statistics file + format (str): Save format, currently only supports "json" + """ + save_path = Path(save_path) + save_path.parent.mkdir(parents=True, exist_ok=True) + + # Build the data structure to save + statistics_data = {} + + # Collect actually used keys from all datasets + all_used_action_keys = set() + all_used_state_keys = set() + + for dataset in self.datasets: + used_action_keys, used_state_keys = get_used_modality_keys(dataset.modality_keys) + all_used_action_keys.update(used_action_keys) + all_used_state_keys.update(used_state_keys) + + # Organize statistics by tag + for tag, merged_metadata in self.merged_metadata.items(): + tag_stats = {} + + # Process action statistics + if hasattr(merged_metadata.statistics, 'action') and merged_metadata.statistics.action: + action_stats = merged_metadata.statistics.action + + # Filter and reorder keys + non_gripper_keys = [] + gripper_keys = [] + + for key in action_stats.keys(): + if key in all_used_action_keys: + if "gripper" in key.lower(): + gripper_keys.append(key) + else: + non_gripper_keys.append(key) + + reordered_keys = non_gripper_keys + gripper_keys + + filtered_action_stats = {} + for key in reordered_keys: + filtered_action_stats[key] = action_stats[key] + + if filtered_action_stats: + combined_action_stats = combine_modality_stats(filtered_action_stats) + + mask = generate_action_mask_for_used_keys( + merged_metadata.modalities.action, filtered_action_stats.keys() + ) + combined_action_stats["mask"] = mask + + tag_stats["action"] = combined_action_stats + + # Process state statistics + if hasattr(merged_metadata.statistics, 'state') and merged_metadata.statistics.state: + state_stats = merged_metadata.statistics.state + + # Filter and reorder keys + non_gripper_keys = [] + gripper_keys = [] + + for key in state_stats.keys(): + if key in all_used_state_keys: + if "gripper" in key.lower(): + gripper_keys.append(key) + else: + non_gripper_keys.append(key) + + reordered_keys = non_gripper_keys + gripper_keys + + filtered_state_stats = {} + for key in reordered_keys: + filtered_state_stats[key] = state_stats[key] + + if filtered_state_stats: + combined_state_stats = combine_modality_stats(filtered_state_stats) + tag_stats["state"] = combined_state_stats + + # Add dataset counts + tag_stats.update(self._get_dataset_counts(tag)) + + statistics_data[tag] = tag_stats + + # Save file + if format.lower() == "json": + if not str(save_path).endswith('.json'): + save_path = save_path.with_suffix('.json') + with open(save_path, 'w', encoding='utf-8') as f: + json.dump(statistics_data, f, indent=2, ensure_ascii=False) + else: + raise ValueError(f"Unsupported format: {format}. Currently only 'json' is supported.") + + print(f"Merged dataset statistics saved to: {save_path}") + print(f"Used action keys (reordered): {list(all_used_action_keys)}") + print(f"Used state keys (reordered): {list(all_used_state_keys)}") + + def _combine_modality_stats(self, modality_stats: dict) -> dict: + """Backward compatibility wrapper.""" + return combine_modality_stats(modality_stats) + + def _generate_action_mask_for_used_keys(self, action_modalities: dict, used_action_keys_ordered) -> list[bool]: + """Backward compatibility wrapper.""" + return generate_action_mask_for_used_keys(action_modalities, used_action_keys_ordered) + + def _get_dataset_counts(self, tag: str) -> dict: + """ + Get dataset count information for specified tag. + + Args: + tag (str): embodiment tag + + Returns: + dict: Dictionary containing num_transitions and num_trajectories + """ + num_transitions = 0 + num_trajectories = 0 + + # Count dataset information belonging to this tag + for dataset in self.datasets: + if dataset.tag == tag: + num_transitions += len(dataset) + num_trajectories += len(dataset.trajectory_ids) + + return { + "num_transitions": num_transitions, + "num_trajectories": num_trajectories + } + + @classmethod + def load_merged_statistics(cls, load_path: Path | str) -> dict: + """ + Load merged dataset statistics from file. + + Args: + load_path (Path | str): Path to the statistics file + + Returns: + dict: Dictionary containing merged statistics + """ + load_path = Path(load_path) + if not load_path.exists(): + raise FileNotFoundError(f"Statistics file not found: {load_path}") + + if load_path.suffix.lower() == '.json': + with open(load_path, 'r', encoding='utf-8') as f: + return json.load(f) + elif load_path.suffix.lower() == '.pkl': + import pickle + with open(load_path, 'rb') as f: + return pickle.load(f) + else: + raise ValueError(f"Unsupported file format: {load_path.suffix}") + + def apply_cached_statistics(self, cached_statistics: dict) -> None: + """ + Apply cached statistics to avoid recomputation. + + Args: + cached_statistics (dict): Statistics loaded from file + """ + # Validate that cached statistics match current datasets + if "metadata" in cached_statistics: + cached_dataset_names = set(cached_statistics["metadata"]["dataset_names"]) + current_dataset_names = set(dataset.dataset_name for dataset in self.datasets) + + if cached_dataset_names != current_dataset_names: + print("Warning: Cached statistics dataset names don't match current datasets.") + print(f"Cached: {cached_dataset_names}") + print(f"Current: {current_dataset_names}") + return + + # Apply cached statistics + self.merged_metadata = {} + for tag, stats_data in cached_statistics.items(): + if tag == "metadata": # Skip metadata field + continue + + # Convert back to DatasetMetadata format + metadata_dict = { + "embodiment_tag": tag, + "statistics": { + "action": {}, + "state": {} + }, + "modalities": {} + } + + # Convert action statistics back + if "action" in stats_data: + action_data = stats_data["action"] + # This is simplified - you may need to split back to sub-keys + metadata_dict["statistics"]["action"] = action_data + + # Convert state statistics back + if "state" in stats_data: + state_data = stats_data["state"] + metadata_dict["statistics"]["state"] = state_data + + self.merged_metadata[tag] = DatasetMetadata.model_validate(metadata_dict) + + # Update transforms metadata for each dataset + for dataset in self.datasets: + if dataset.tag in self.merged_metadata: + dataset.set_transforms_metadata(self.merged_metadata[dataset.tag]) + + print(f"Applied cached statistics for {len(self.merged_metadata)} embodiment tags.") diff --git a/src/InternVLA-M1/InternVLA/dataloader/gr00t_lerobot/embodiment_tags.py b/src/InternVLA-M1/InternVLA/dataloader/gr00t_lerobot/embodiment_tags.py new file mode 100644 index 0000000000000000000000000000000000000000..6f6c7993349446e5720f92d0ea9aec3727ffc5b7 --- /dev/null +++ b/src/InternVLA-M1/InternVLA/dataloader/gr00t_lerobot/embodiment_tags.py @@ -0,0 +1,76 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from enum import Enum + + +class EmbodimentTag(Enum): + GR1 = "gr1" + """ + The GR1 dataset. + """ + + OXE_DROID = "oxe_droid" + """ + The OxE Droid dataset. + """ + + OXE_BRIDGE = "oxe_bridge" + """ + The OxE Bridge dataset. + """ + + OXE_RT1 = "oxe_rt1" + """ + The OxE RT-1 dataset. + """ + + AGIBOT_GENIE1 = "agibot_genie1" + """ + The AgiBot Genie-1 with gripper dataset. + """ + + NEW_EMBODIMENT = "new_embodiment" + """ + Any new embodiment for finetuning. + """ + + FRANKA = 'franka' + """ + The Franka Emika Panda robot. + """ + +# Embodiment tag string: to projector index in the Action Expert Module +EMBODIMENT_TAG_MAPPING = { + EmbodimentTag.NEW_EMBODIMENT.value: 31, + EmbodimentTag.OXE_DROID.value: 17, + EmbodimentTag.OXE_BRIDGE.value: 18, + EmbodimentTag.OXE_RT1.value: 19, + EmbodimentTag.AGIBOT_GENIE1.value: 26, + EmbodimentTag.GR1.value: 24, + EmbodimentTag.FRANKA.value: 25, +} + +# Robot type to embodiment tag mapping +ROBOT_TYPE_TO_EMBODIMENT_TAG = { + "libero_franka": EmbodimentTag.FRANKA, + "oxe_droid": EmbodimentTag.OXE_DROID, + "oxe_bridge": EmbodimentTag.OXE_BRIDGE, + "oxe_rt1": EmbodimentTag.OXE_RT1, + "demo_sim_franka_delta_joints": EmbodimentTag.FRANKA, + "custom_robot_config": EmbodimentTag.NEW_EMBODIMENT, + "humanoid_config": EmbodimentTag.NEW_EMBODIMENT, + "humanoid_sim_config": EmbodimentTag.NEW_EMBODIMENT, +} diff --git a/src/InternVLA-M1/InternVLA/dataloader/gr00t_lerobot/mixtures.py b/src/InternVLA-M1/InternVLA/dataloader/gr00t_lerobot/mixtures.py new file mode 100644 index 0000000000000000000000000000000000000000..b024153923aa106524ab11decd002069bd624660 --- /dev/null +++ b/src/InternVLA-M1/InternVLA/dataloader/gr00t_lerobot/mixtures.py @@ -0,0 +1,62 @@ +""" +mixtures.py + +Defines a registry of dataset mixtures and weights for the Open-X Embodiment Datasets. Each dataset is associated with +a float "sampling weight" +""" + +from typing import Dict, List, Tuple + +DATASET_NAMED_MIXTURES = { + + "libero_all": [ + ("libero_object_no_noops_1.0.0_lerobot", 1.0, "libero_franka"), + ("libero_goal_no_noops_1.0.0_lerobot", 1.0, "libero_franka"), + ("libero_spatial_no_noops_1.0.0_lerobot", 1.0, "libero_franka"), + ("libero_90_no_noops_lerobot", 1.0, "libero_franka"), + ("libero_10_no_noops_1.0.0_lerobot", 1.0, "libero_franka"), + ], + "libero_goal": [ + ("libero_goal_no_noops_1.0.0_lerobot", 1.0, "libero_franka"), + ], + "libero_object": [ + ("libero_object_no_noops_1.0.0_lerobot", 1.0, "libero_franka"), + ], + "libero_spatial": [ + ("libero_spatial_no_noops_1.0.0_lerobot", 1.0, "libero_franka"), + ], + "libero_10": [ + ("libero_10_no_noops_1.0.0_lerobot", 1.0, "libero_franka"), + ], + "libero_90": [ + ("libero_90_no_noops_lerobot", 1.0, "libero_franka"), + ("libero_90_no_noops_lerobot", 1.0, "libero_ur5"), + ], + + "bridge": [ + ("bridge_orig_1.0.0_lerobot", 1.0, "oxe_bridge"), + ], + "bridge_rt_1": [ + ("bridge_orig_1.0.0_lerobot", 1.0, "oxe_bridge"), + ("fractal20220817_data_0.1.0_lerobot", 1.0, "oxe_rt1"), + ], + + "demo_sim_pick_place": [ + ("sim_pick_place", 1.0, "demo_sim_franka_delta_joints"), + ], + + "custom_dataset": [ + ("custom_dataset_name", 1.0, "custom_robot_config"), + ], + "custom_dataset_2": [ + ("custom_dataset_name_1", 1.0, "custom_robot_config"), + ("custom_dataset_name_2", 1.0, "custom_robot_config"), + ], + "humanoid_": [ + ("g1-fullbody-lerobot", 1.0, "humanoid_config"), + ], + "humanoid_sim": [ + ("g1-fullbody-lerobot", 1.0, "humanoid_sim_config"), + ], + +} diff --git a/src/InternVLA-M1/InternVLA/dataloader/gr00t_lerobot/schema.py b/src/InternVLA-M1/InternVLA/dataloader/gr00t_lerobot/schema.py new file mode 100644 index 0000000000000000000000000000000000000000..64519e56a3c59e5d08c8f8f6370f640061859b4d --- /dev/null +++ b/src/InternVLA-M1/InternVLA/dataloader/gr00t_lerobot/schema.py @@ -0,0 +1,221 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from enum import Enum +from typing import Optional + +from numpydantic import NDArray +from pydantic import BaseModel, Field, field_serializer + +from .embodiment_tags import EmbodimentTag + +# Common schema + + +class RotationType(Enum): + """Type of rotation representation""" + + AXIS_ANGLE = "axis_angle" + QUATERNION = "quaternion" + ROTATION_6D = "rotation_6d" + MATRIX = "matrix" + EULER_ANGLES_RPY = "euler_angles_rpy" + EULER_ANGLES_RYP = "euler_angles_ryp" + EULER_ANGLES_PRY = "euler_angles_pry" + EULER_ANGLES_PYR = "euler_angles_pyr" + EULER_ANGLES_YRP = "euler_angles_yrp" + EULER_ANGLES_YPR = "euler_angles_ypr" + + +# LeRobot schema + + +class LeRobotModalityField(BaseModel): + """Metadata for a LeRobot modality field.""" + + original_key: Optional[str] = Field( + default=None, + description="The original key of the modality in the LeRobot dataset", + ) + + +class LeRobotStateActionMetadata(LeRobotModalityField): + """Metadata for a LeRobot modality.""" + + start: int = Field( + ..., + description="The start index of the modality in the concatenated state/action vector", + ) + end: int = Field( + ..., + description="The end index of the modality in the concatenated state/action vector", + ) + rotation_type: Optional[RotationType] = Field( + default=None, description="The type of rotation for the modality" + ) + absolute: bool = Field(default=True, description="Whether the modality is absolute") + dtype: str = Field( + default="float64", + description="The data type of the modality. Defaults to float64.", + ) + range: Optional[tuple[float, float]] = Field( + default=None, + description="The range of the modality, if applicable. Defaults to None.", + ) + original_key: Optional[str] = Field( + default=None, + description="The original key of the modality in the LeRobot dataset.", + ) + + +class LeRobotStateMetadata(LeRobotStateActionMetadata): + """Metadata for a LeRobot state modality.""" + + original_key: Optional[str] = Field( + default="observation.state", # LeRobot convention for states + description="The original key of the state modality in the LeRobot dataset", + ) + + +class LeRobotActionMetadata(LeRobotStateActionMetadata): + """Metadata for a LeRobot action modality.""" + + original_key: Optional[str] = Field( + default="action", # LeRobot convention for actions + description="The original key of the action modality in the LeRobot dataset", + ) + + +class LeRobotModalityMetadata(BaseModel): + """Metadata for a LeRobot modality.""" + + state: dict[str, LeRobotStateMetadata] = Field( + ..., + description="The metadata for the state modality. The keys are the names of each split of the state vector.", + ) + action: dict[str, LeRobotActionMetadata] = Field( + ..., + description="The metadata for the action modality. The keys are the names of each split of the action vector.", + ) + video: dict[str, LeRobotModalityField] = Field( + ..., + description="The metadata for the video modality. The keys are the new names of each video modality.", + ) + annotation: Optional[dict[str, LeRobotModalityField]] = Field( + default=None, + description="The metadata for the annotation modality. The keys are the new names of each annotation modality.", + ) + + def get_key_meta(self, key: str) -> LeRobotModalityField: + """Get the metadata for a key in the LeRobot modality metadata. + + Args: + key (str): The key to get the metadata for. + + Returns: + LeRobotModalityField: The metadata for the key. + + Example: + lerobot_modality_meta = LeRobotModalityMetadata.model_validate(U.load_json(modality_meta_path)) + lerobot_modality_meta.get_key_meta("state.joint_shoulder_y") + lerobot_modality_meta.get_key_meta("video.main_camera") + lerobot_modality_meta.get_key_meta("annotation.human.action.task_description") + """ + split_key = key.split(".") + modality = split_key[0] + subkey = ".".join(split_key[1:]) + if modality == "state": + if subkey not in self.state: + raise ValueError( + f"Key: {key}, state key {subkey} not found in metadata, available state keys: {self.state.keys()}" + ) + return self.state[subkey] + elif modality == "action": + if subkey not in self.action: + raise ValueError( + f"Key: {key}, action key {subkey} not found in metadata, available action keys: {self.action.keys()}" + ) + return self.action[subkey] + elif modality == "video": + if subkey not in self.video: + raise ValueError( + f"Key: {key}, video key {subkey} not found in metadata, available video keys: {self.video.keys()}" + ) + return self.video[subkey] + elif modality == "annotation": + assert ( + self.annotation is not None + ), "Trying to get annotation metadata for a dataset with no annotations" + if subkey not in self.annotation: + raise ValueError( + f"Key: {key}, annotation key {subkey} not found in metadata, available annotation keys: {self.annotation.keys()}" + ) + return self.annotation[subkey] + else: + raise ValueError(f"Key: {key}, unexpected modality: {modality}") + + +# Dataset schema (parsed from LeRobot schema and simplified) + + +class DatasetStatisticalValues(BaseModel): + max: NDArray = Field(..., description="Maximum values") + min: NDArray = Field(..., description="Minimum values") + mean: NDArray = Field(..., description="Mean values") + std: NDArray = Field(..., description="Standard deviation") + q01: NDArray = Field(..., description="1st percentile values") + q99: NDArray = Field(..., description="99th percentile values") + + @field_serializer("*", when_used="json") + def serialize_ndarray(self, v: NDArray) -> list[float]: + return v.tolist() # type: ignore + + +class DatasetStatistics(BaseModel): + state: dict[str, DatasetStatisticalValues] = Field(..., description="Statistics of the state") + action: dict[str, DatasetStatisticalValues] = Field(..., description="Statistics of the action") + + +class VideoMetadata(BaseModel): + """Metadata of the video modality""" + + resolution: tuple[int, int] = Field(..., description="Resolution of the video") + channels: int = Field(..., description="Number of channels in the video", gt=0) + fps: float = Field(..., description="Frames per second", gt=0) + + +class StateActionMetadata(BaseModel): + absolute: bool = Field(..., description="Whether the state or action is absolute") + rotation_type: Optional[RotationType] = Field(None, description="Type of rotation, if any") + shape: tuple[int, ...] = Field(..., description="Shape of the state or action") + continuous: bool = Field(..., description="Whether the state or action is continuous") + + +class DatasetModalities(BaseModel): + video: dict[str, VideoMetadata] = Field(..., description="Metadata of the video") + state: dict[str, StateActionMetadata] = Field(..., description="Metadata of the state") + action: dict[str, StateActionMetadata] = Field(..., description="Metadata of the action") + + +class DatasetMetadata(BaseModel): + """Metadata of the trainable dataset + + Changes: + - Update to use the new RawCommitHashMetadataMetadata_V1_2 + """ + + statistics: DatasetStatistics = Field(..., description="Statistics of the dataset") + modalities: DatasetModalities = Field(..., description="Metadata of the modalities") + embodiment_tag: EmbodimentTag = Field(..., description="Embodiment tag of the dataset") \ No newline at end of file diff --git a/src/InternVLA-M1/InternVLA/dataloader/gr00t_lerobot/transform/__init__.py b/src/InternVLA-M1/InternVLA/dataloader/gr00t_lerobot/transform/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..bf22b5e4ca3d1c7937a25234cbf08e2644593587 --- /dev/null +++ b/src/InternVLA-M1/InternVLA/dataloader/gr00t_lerobot/transform/__init__.py @@ -0,0 +1,41 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from .base import ( + ComposedModalityTransform, + InvertibleModalityTransform, + ModalityTransform, +) +from .concat import ConcatTransform +# from .state_action import ( +# StateActionDropout, +# StateActionPerturbation, +# StateActionSinCosTransform, +# StateActionToTensor, +# StateActionTransform, +# ) +from .video import ( + VideoColorJitter, + VideoCrop, + VideoGrayscale, + VideoHorizontalFlip, + VideoRandomGrayscale, + VideoRandomPosterize, + VideoRandomRotation, + VideoResize, + VideoToNumpy, + VideoToTensor, + VideoTransform, +) diff --git a/src/InternVLA-M1/InternVLA/dataloader/gr00t_lerobot/transform/base.py b/src/InternVLA-M1/InternVLA/dataloader/gr00t_lerobot/transform/base.py new file mode 100644 index 0000000000000000000000000000000000000000..aac88559af98fa23f34fbb9135775d0819c281ef --- /dev/null +++ b/src/InternVLA-M1/InternVLA/dataloader/gr00t_lerobot/transform/base.py @@ -0,0 +1,134 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from abc import ABC, abstractmethod +from typing import Any + +from pydantic import BaseModel, ConfigDict, Field, PrivateAttr + +from ..schema import DatasetMetadata + + +class ModalityTransform(BaseModel, ABC): + """ + Abstract class for transforming data modalities, e.g. video frame augmentation or action normalization. + """ + + apply_to: list[str] = Field(..., description="The keys to apply the transform to.") + training: bool = Field( + default=True, description="Whether to apply the transform in training mode." + ) + _dataset_metadata: DatasetMetadata | None = PrivateAttr(default=None) + + model_config = ConfigDict(arbitrary_types_allowed=True) + + @property + def dataset_metadata(self) -> DatasetMetadata: + assert ( + self._dataset_metadata is not None + ), "Dataset metadata is not set. Please call set_metadata() before calling apply()." + return self._dataset_metadata + + @dataset_metadata.setter + def dataset_metadata(self, value: DatasetMetadata): + self._dataset_metadata = value + + def set_metadata(self, dataset_metadata: DatasetMetadata): + """ + Set the dataset metadata. This is useful for transforms that need to know the dataset metadata, e.g. to normalize actions. + Subclasses can override this method if they need to do something more complex. + """ + self.dataset_metadata = dataset_metadata + + def __call__(self, data: dict[str, Any]) -> dict[str, Any]: + """Apply the transformation to the data corresponding to target_keys and return the processed data. + + Args: + data (dict[str, Any]): The data to transform. + example: data = { + "video.image_side_0": np.ndarray, + "action.eef_position": np.ndarray, + ... + } + + Returns: + dict[str, Any]: The transformed data. + example: transformed_data = { + "video.image_side_0": np.ndarray, + "action.eef_position": torch.Tensor, # Normalized and converted to tensor + ... + } + """ + return self.apply(data) + + @abstractmethod + def apply(self, data: dict[str, Any]) -> dict[str, Any]: + """Apply the transformation to the data corresponding to keys matching the `apply_to` regular expression and return the processed data.""" + + def train(self): + self.training = True + + def eval(self): + self.training = False + + +class InvertibleModalityTransform(ModalityTransform): + @abstractmethod + def unapply(self, data: dict[str, Any]) -> dict[str, Any]: + """Reverse the transformation to the data corresponding to keys matching the `apply_to` regular expression and return the processed data.""" + + +class ComposedModalityTransform(ModalityTransform): + """Compose multiple modality transforms.""" + + transforms: list[ModalityTransform] = Field(..., description="The transforms to compose.") + apply_to: list[str] = Field( + default_factory=list, description="Will be ignored for composed transforms." + ) + training: bool = Field( + default=True, description="Whether to apply the transform in training mode." + ) + + model_config = ConfigDict(arbitrary_types_allowed=True, from_attributes=True) + + def set_metadata(self, dataset_metadata: DatasetMetadata): + for transform in self.transforms: + transform.set_metadata(dataset_metadata) + + def apply(self, data: dict[str, Any]) -> dict[str, Any]: + for i, transform in enumerate(self.transforms): + try: + data = transform(data) + except Exception as e: + raise ValueError(f"Error applying transform {i} to data: {e}") from e + return data + + def unapply(self, data: dict[str, Any]) -> dict[str, Any]: + for i, transform in enumerate(reversed(self.transforms)): + if isinstance(transform, InvertibleModalityTransform): + try: + data = transform.unapply(data) + except Exception as e: + step = len(self.transforms) - i - 1 + raise ValueError(f"Error unapplying transform {step} to data: {e}") from e + return data + + def train(self): + for transform in self.transforms: + transform.train() + + def eval(self): + for transform in self.transforms: + transform.eval() diff --git a/src/InternVLA-M1/InternVLA/dataloader/gr00t_lerobot/transform/concat.py b/src/InternVLA-M1/InternVLA/dataloader/gr00t_lerobot/transform/concat.py new file mode 100644 index 0000000000000000000000000000000000000000..cf8eea4c77fc163ecdb0d25aeca26a2cde99f8c4 --- /dev/null +++ b/src/InternVLA-M1/InternVLA/dataloader/gr00t_lerobot/transform/concat.py @@ -0,0 +1,215 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import Optional + +import numpy as np +import torch +from pydantic import Field + +from ..schema import DatasetMetadata, StateActionMetadata +from .base import InvertibleModalityTransform + + +class ConcatTransform(InvertibleModalityTransform): + """ + Concatenate the keys according to specified order. + """ + + # -- We inherit from ModalityTransform, so we keep apply_to as well -- + apply_to: list[str] = Field( + default_factory=list, description="Not used in this transform, kept for compatibility." + ) + + video_concat_order: list[str] = Field( + ..., + description="Concatenation order for each video modality. " + "Format: ['video.ego_view_pad_res224_freq20', ...]", + ) + + state_concat_order: Optional[list[str]] = Field( + default=None, + description="Concatenation order for each state modality. " + "Format: ['state.position', 'state.velocity', ...].", + ) + + action_concat_order: Optional[list[str]] = Field( + default=None, + description="Concatenation order for each action modality. " + "Format: ['action.position', 'action.velocity', ...].", + ) + + action_dims: dict[str, int] = Field( + default_factory=dict, + description="The dimensions of the action keys.", + ) + state_dims: dict[str, int] = Field( + default_factory=dict, + description="The dimensions of the state keys.", + ) + + def model_dump(self, *args, **kwargs): + if kwargs.get("mode", "python") == "json": + include = { + "apply_to", + "video_concat_order", + "state_concat_order", + "action_concat_order", + } + else: + include = kwargs.pop("include", None) + + return super().model_dump(*args, include=include, **kwargs) + + def apply(self, data: dict) -> dict: + grouped_keys = {} + for key in data.keys(): + try: + modality, _ = key.split(".") + except: # noqa: E722 + ### Handle language annotation special case + if "annotation" in key: + modality = "language" + else: + modality = "others" + if modality not in grouped_keys: + grouped_keys[modality] = [] + grouped_keys[modality].append(key) + + if "video" in grouped_keys: + # Check if keys in video_concat_order, state_concat_order, action_concat_order are + # ineed contained in the data. If not, then the keys are misspecified + video_keys = grouped_keys["video"] + assert self.video_concat_order is not None, f"{self.video_concat_order=}, {video_keys=}" + assert all( + item in video_keys for item in self.video_concat_order + ), f"keys in video_concat_order are misspecified, \n{video_keys=}, \n{self.video_concat_order=}" + + # Process each video view + unsqueezed_videos = [] + for video_key in self.video_concat_order: + video_data = data.pop(video_key) + unsqueezed_video = np.expand_dims( + video_data, axis=-4 + ) # [..., H, W, C] -> [..., 1, H, W, C] + unsqueezed_videos.append(unsqueezed_video) + # Concatenate along the new axis + unsqueezed_video = np.concatenate(unsqueezed_videos, axis=-4) # [..., V, H, W, C] + + # Video + data["video"] = unsqueezed_video + + # "state" + if "state" in grouped_keys: + state_keys = grouped_keys["state"] + assert self.state_concat_order is not None, f"{self.state_concat_order=}" + assert all( + item in state_keys for item in self.state_concat_order + ), f"keys in state_concat_order are misspecified, \n{state_keys=}, \n{self.state_concat_order=}" + # Check the state dims + for key in self.state_concat_order: + target_shapes = [self.state_dims[key]] + if self.is_rotation_key(key): + target_shapes.append(6) # Allow for rotation_6d + # if key in ["state.right_arm", "state.right_hand"]: + target_shapes.append(self.state_dims[key] * 2) # Allow for sin-cos transform + assert ( + data[key].shape[-1] in target_shapes + ), f"State dim mismatch for {key=}, {data[key].shape[-1]=}, {target_shapes=}" + # Concatenate the state keys + # We'll have StateActionToTensor before this transform, so here we use torch.cat + data["state"] = torch.cat( + [data.pop(key) for key in self.state_concat_order], dim=-1 + ) # [T, D_state] + + if "action" in grouped_keys: + action_keys = grouped_keys["action"] + assert self.action_concat_order is not None, f"{self.action_concat_order=}" + # Check if all keys in concat_order are present + assert set(self.action_concat_order) == set( + action_keys + ), f"{set(self.action_concat_order)=}, {set(action_keys)=}" + # Record the action dims + for key in self.action_concat_order: + target_shapes = [self.action_dims[key]] + if self.is_rotation_key(key): + target_shapes.append(3) # Allow for axis angle + assert ( + self.action_dims[key] == data[key].shape[-1] + ), f"Action dim mismatch for {key=}, {self.action_dims[key]=}, {data[key].shape[-1]=}" + # Concatenate the action keys + # We'll have StateActionToTensor before this transform, so here we use torch.cat + data["action"] = torch.cat( + [data.pop(key) for key in self.action_concat_order], dim=-1 + ) # [T, D_action] + + return data + + def unapply(self, data: dict) -> dict: + start_dim = 0 + assert "action" in data, f"{data.keys()=}" + # For those dataset without actions (LAPA), we'll never run unapply + assert self.action_concat_order is not None, f"{self.action_concat_order=}" + action_tensor = data.pop("action") + for key in self.action_concat_order: + if key not in self.action_dims: + raise ValueError(f"Action dim {key} not found in action_dims.") + end_dim = start_dim + self.action_dims[key] + data[key] = action_tensor[..., start_dim:end_dim] + start_dim = end_dim + if "state" in data: + assert self.state_concat_order is not None, f"{self.state_concat_order=}" + start_dim = 0 + state_tensor = data.pop("state") + for key in self.state_concat_order: + end_dim = start_dim + self.state_dims[key] + data[key] = state_tensor[..., start_dim:end_dim] + start_dim = end_dim + return data + + def __call__(self, data: dict) -> dict: + return self.apply(data) + + def get_modality_metadata(self, key: str) -> StateActionMetadata: + modality, subkey = key.split(".") + assert self.dataset_metadata is not None, "Metadata not set" + modality_config = getattr(self.dataset_metadata.modalities, modality) + assert subkey in modality_config, f"{subkey=} not found in {modality_config=}" + assert isinstance( + modality_config[subkey], StateActionMetadata + ), f"Expected {StateActionMetadata} for {subkey=}, got {type(modality_config[subkey])=}" + return modality_config[subkey] + + def get_state_action_dims(self, key: str) -> int: + """Get the dimension of a state or action key from the dataset metadata.""" + modality_config = self.get_modality_metadata(key) + shape = modality_config.shape + assert len(shape) == 1, f"{shape=}" + return shape[0] + + def is_rotation_key(self, key: str) -> bool: + modality_config = self.get_modality_metadata(key) + return modality_config.rotation_type is not None + + def set_metadata(self, dataset_metadata: DatasetMetadata): + """Set the metadata and compute the dimensions of the state and action keys.""" + super().set_metadata(dataset_metadata) + # Pre-compute the dimensions of the state and action keys + if self.action_concat_order is not None: + for key in self.action_concat_order: + self.action_dims[key] = self.get_state_action_dims(key) + if self.state_concat_order is not None: + for key in self.state_concat_order: + self.state_dims[key] = self.get_state_action_dims(key) diff --git a/src/InternVLA-M1/InternVLA/dataloader/gr00t_lerobot/transform/state_action.py b/src/InternVLA-M1/InternVLA/dataloader/gr00t_lerobot/transform/state_action.py new file mode 100644 index 0000000000000000000000000000000000000000..b4ff07877d00bf025f04a55391e58006f70d55fa --- /dev/null +++ b/src/InternVLA-M1/InternVLA/dataloader/gr00t_lerobot/transform/state_action.py @@ -0,0 +1,606 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import functools +import random +from typing import Any, ClassVar + +import numpy as np +import pytorch3d.transforms as pt +import torch +from pydantic import Field, PrivateAttr, field_validator, model_validator + +from ..schema import DatasetMetadata, RotationType, StateActionMetadata +from .base import InvertibleModalityTransform, ModalityTransform + + +class RotationTransform: + """Adapted from https://github.com/real-stanford/diffusion_policy/blob/548a52bbb105518058e27bf34dcf90bf6f73681a/diffusion_policy/model/common/rotation_transformer.py""" + + valid_reps = ["axis_angle", "euler_angles", "quaternion", "rotation_6d", "matrix"] + + def __init__(self, from_rep="axis_angle", to_rep="rotation_6d"): + """ + Valid representations + + Always use matrix as intermediate representation. + """ + if from_rep.startswith("euler_angles"): + from_convention = from_rep.split("_")[-1] + from_rep = "euler_angles" + from_convention = from_convention.replace("r", "X").replace("p", "Y").replace("y", "Z") + else: + from_convention = None + if to_rep.startswith("euler_angles"): + to_convention = to_rep.split("_")[-1] + to_rep = "euler_angles" + to_convention = to_convention.replace("r", "X").replace("p", "Y").replace("y", "Z") + else: + to_convention = None + assert from_rep != to_rep, f"from_rep and to_rep cannot be the same: {from_rep}" + assert from_rep in self.valid_reps, f"Invalid from_rep: {from_rep}" + assert to_rep in self.valid_reps, f"Invalid to_rep: {to_rep}" + + forward_funcs = list() + inverse_funcs = list() + + if from_rep != "matrix": + funcs = [getattr(pt, f"{from_rep}_to_matrix"), getattr(pt, f"matrix_to_{from_rep}")] + if from_convention is not None: + funcs = [functools.partial(func, convention=from_convention) for func in funcs] + forward_funcs.append(funcs[0]) + inverse_funcs.append(funcs[1]) + + if to_rep != "matrix": + funcs = [getattr(pt, f"matrix_to_{to_rep}"), getattr(pt, f"{to_rep}_to_matrix")] + if to_convention is not None: + funcs = [functools.partial(func, convention=to_convention) for func in funcs] + forward_funcs.append(funcs[0]) + inverse_funcs.append(funcs[1]) + + inverse_funcs = inverse_funcs[::-1] + + self.forward_funcs = forward_funcs + self.inverse_funcs = inverse_funcs + + @staticmethod + def _apply_funcs(x: torch.Tensor, funcs: list) -> torch.Tensor: + assert isinstance(x, torch.Tensor) + for func in funcs: + x = func(x) + return x + + def forward(self, x: torch.Tensor) -> torch.Tensor: + assert isinstance( + x, torch.Tensor + ), f"Unexpected input type: {type(x)}. Expected type: {torch.Tensor}" + return self._apply_funcs(x, self.forward_funcs) + + def inverse(self, x: torch.Tensor) -> torch.Tensor: + assert isinstance( + x, torch.Tensor + ), f"Unexpected input type: {type(x)}. Expected type: {torch.Tensor}" + return self._apply_funcs(x, self.inverse_funcs) + + +class Normalizer: + valid_modes = ["q99", "mean_std", "min_max", "binary"] + + def __init__(self, mode: str, statistics: dict): + self.mode = mode + self.statistics = statistics + for key, value in self.statistics.items(): + self.statistics[key] = torch.tensor(value) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + assert isinstance( + x, torch.Tensor + ), f"Unexpected input type: {type(x)}. Expected type: {torch.Tensor}" + + # Normalize the tensor + if self.mode == "q99": + # Range of q99 is [-1, 1] + q01 = self.statistics["q01"].to(x.dtype) + q99 = self.statistics["q99"].to(x.dtype) + + # In the case of q01 == q99, the normalization will be undefined + # So we set the normalized values to the original values + mask = q01 != q99 + normalized = torch.zeros_like(x) + + # Normalize the values where q01 != q99 + # Formula: 2 * (x - q01) / (q99 - q01) - 1 + normalized[..., mask] = (x[..., mask] - q01[..., mask]) / ( + q99[..., mask] - q01[..., mask] + ) + normalized[..., mask] = 2 * normalized[..., mask] - 1 + + # Set the normalized values to the original values where q01 == q99 + normalized[..., ~mask] = x[..., ~mask].to(x.dtype) + + # Clip the normalized values to be between -1 and 1 + normalized = torch.clamp(normalized, -1, 1) + + elif self.mode == "mean_std": + # Range of mean_std is not fixed, but can be positive or negative + mean = self.statistics["mean"].to(x.dtype) + std = self.statistics["std"].to(x.dtype) + + # In the case of std == 0, the normalization will be undefined + # So we set the normalized values to the original values + mask = std != 0 + normalized = torch.zeros_like(x) + + # Normalize the values where std != 0 + # Formula: (x - mean) / std + normalized[..., mask] = (x[..., mask] - mean[..., mask]) / std[..., mask] + + # Set the normalized values to the original values where std == 0 + normalized[..., ~mask] = x[..., ~mask].to(x.dtype) + + elif self.mode == "min_max": + # Range of min_max is [-1, 1] + min = self.statistics["min"].to(x.dtype) + max = self.statistics["max"].to(x.dtype) + + # In the case of min == max, the normalization will be undefined + # So we set the normalized values to the original values + mask = min != max + normalized = torch.zeros_like(x) + + # Normalize the values where min != max + # Formula: 2 * (x - min) / (max - min) - 1 + normalized[..., mask] = (x[..., mask] - min[..., mask]) / ( + max[..., mask] - min[..., mask] + ) + normalized[..., mask] = 2 * normalized[..., mask] - 1 + + # Set the normalized values to the original values where min == max + # normalized[..., ~mask] = x[..., ~mask].to(x.dtype) + # Set the normalized values to 0 where min == max + normalized[..., ~mask] = 0 + + elif self.mode == "scale": + # Range of scale is [0, 1] + min = self.statistics["min"].to(x.dtype) + max = self.statistics["max"].to(x.dtype) + abs_max = torch.max(torch.abs(min), torch.abs(max)) + mask = abs_max != 0 + normalized = torch.zeros_like(x) + normalized[..., mask] = x[..., mask] / abs_max[..., mask] + normalized[..., ~mask] = 0 + + elif self.mode == "binary": + # Range of binary is [0, 1] + normalized = (x > 0.5).to(x.dtype) + else: + raise ValueError(f"Invalid normalization mode: {self.mode}") + + return normalized + + def inverse(self, x: torch.Tensor) -> torch.Tensor: + assert isinstance( + x, torch.Tensor + ), f"Unexpected input type: {type(x)}. Expected type: {torch.Tensor}" + if self.mode == "q99": + q01 = self.statistics["q01"].to(x.dtype) + q99 = self.statistics["q99"].to(x.dtype) + return (x + 1) / 2 * (q99 - q01) + q01 + elif self.mode == "mean_std": + mean = self.statistics["mean"].to(x.dtype) + std = self.statistics["std"].to(x.dtype) + return x * std + mean + elif self.mode == "min_max": + min = self.statistics["min"].to(x.dtype) + max = self.statistics["max"].to(x.dtype) + return (x + 1) / 2 * (max - min) + min + elif self.mode == "binary": + return (x > 0.5).to(x.dtype) + else: + raise ValueError(f"Invalid normalization mode: {self.mode}") + + +class StateActionToTensor(InvertibleModalityTransform): + """ + Transforms states and actions to tensors. + """ + + input_dtypes: dict[str, np.dtype] = Field( + default_factory=dict, description="The input dtypes for each state key." + ) + output_dtypes: dict[str, torch.dtype] = Field( + default_factory=dict, description="The output dtypes for each state key." + ) + + def model_dump(self, *args, **kwargs): + if kwargs.get("mode", "python") == "json": + include = {"apply_to"} + else: + include = kwargs.pop("include", None) + + return super().model_dump(*args, include=include, **kwargs) + + @field_validator("input_dtypes", "output_dtypes", mode="before") + def validate_dtypes(cls, v): + for key, dtype in v.items(): + if isinstance(dtype, str): + if dtype.startswith("torch."): + dtype_split = dtype.split(".")[-1] + v[key] = getattr(torch, dtype_split) + elif dtype.startswith("np.") or dtype.startswith("numpy."): + dtype_split = dtype.split(".")[-1] + v[key] = np.dtype(dtype_split) + else: + raise ValueError(f"Invalid dtype: {dtype}") + return v + + def apply(self, data: dict[str, Any]) -> dict[str, Any]: + for key in self.apply_to: + if key not in data: + continue + value = data[key] + assert isinstance( + value, np.ndarray + ), f"Unexpected input type: {type(value)}. Expected type: {np.ndarray}" + data[key] = torch.from_numpy(value) + if key in self.output_dtypes: + data[key] = data[key].to(self.output_dtypes[key]) + return data + + def unapply(self, data: dict[str, Any]) -> dict[str, Any]: + for key in self.apply_to: + if key not in data: + continue + value = data[key] + assert isinstance( + value, torch.Tensor + ), f"Unexpected input type: {type(value)}. Expected type: {torch.Tensor}" + data[key] = value.numpy() + if key in self.input_dtypes: + data[key] = data[key].astype(self.input_dtypes[key]) + return data + + +class StateActionTransform(InvertibleModalityTransform): + """ + Class for state or action transform. + + Args: + apply_to (list[str]): The keys in the modality to load and transform. + normalization_modes (dict[str, str]): The normalization modes for each state key. + If a state key in apply_to is not present in the dictionary, it will not be normalized. + target_rotations (dict[str, str]): The target representations for each state key. + If a state key in apply_to is not present in the dictionary, it will not be rotated. + """ + + # Configurable attributes + apply_to: list[str] = Field(..., description="The keys in the modality to load and transform.") + normalization_modes: dict[str, str] = Field( + default_factory=dict, description="The normalization modes for each state key." + ) + target_rotations: dict[str, str] = Field( + default_factory=dict, description="The target representations for each state key." + ) + normalization_statistics: dict[str, dict] = Field( + default_factory=dict, description="The statistics for each state key." + ) + modality_metadata: dict[str, StateActionMetadata] = Field( + default_factory=dict, description="The modality metadata for each state key." + ) + + # Model variables + _rotation_transformers: dict[str, RotationTransform] = PrivateAttr(default_factory=dict) + _normalizers: dict[str, Normalizer] = PrivateAttr(default_factory=dict) + _input_dtypes: dict[str, np.dtype | torch.dtype] = PrivateAttr(default_factory=dict) + + # Model constants + _DEFAULT_MIN_MAX_STATISTICS: ClassVar[dict] = { + "rotation_6d": { + "min": [-1, -1, -1, -1, -1, -1], + "max": [1, 1, 1, 1, 1, 1], + }, + "euler_angles": { + "min": [-np.pi, -np.pi, -np.pi], + "max": [np.pi, np.pi, np.pi], + }, + "quaternion": { + "min": [-1, -1, -1, -1], + "max": [1, 1, 1, 1], + }, + "axis_angle": { + "min": [-np.pi, -np.pi, -np.pi], + "max": [np.pi, np.pi, np.pi], + }, + } + + def model_dump(self, *args, **kwargs): + if kwargs.get("mode", "python") == "json": + include = {"apply_to", "normalization_modes", "target_rotations"} + else: + include = kwargs.pop("include", None) + + return super().model_dump(*args, include=include, **kwargs) + + @field_validator("modality_metadata", mode="before") + def validate_modality_metadata(cls, v): + for modality_key, config in v.items(): + if isinstance(config, dict): + config = StateActionMetadata.model_validate(config) + else: + assert isinstance( + config, StateActionMetadata + ), f"Invalid source rotation config: {config}" + v[modality_key] = config + return v + + @model_validator(mode="after") + def validate_normalization_statistics(self): + for modality_key, normalization_statistics in self.normalization_statistics.items(): + if modality_key in self.normalization_modes: + normalization_mode = self.normalization_modes[modality_key] + if normalization_mode == "min_max": + assert ( + "min" in normalization_statistics and "max" in normalization_statistics + ), f"Min and max statistics are required for min_max normalization, but got {normalization_statistics}" + assert len(normalization_statistics["min"]) == len( + normalization_statistics["max"] + ), f"Min and max statistics must have the same length, but got {normalization_statistics['min']} and {normalization_statistics['max']}" + elif normalization_mode == "mean_std": + assert ( + "mean" in normalization_statistics and "std" in normalization_statistics + ), f"Mean and std statistics are required for mean_std normalization, but got {normalization_statistics}" + assert len(normalization_statistics["mean"]) == len( + normalization_statistics["std"] + ), f"Mean and std statistics must have the same length, but got {normalization_statistics['mean']} and {normalization_statistics['std']}" + elif normalization_mode == "q99": + assert ( + "q01" in normalization_statistics and "q99" in normalization_statistics + ), f"q01 and q99 statistics are required for q99 normalization, but got {normalization_statistics}" + assert len(normalization_statistics["q01"]) == len( + normalization_statistics["q99"] + ), f"q01 and q99 statistics must have the same length, but got {normalization_statistics['q01']} and {normalization_statistics['q99']}" + elif normalization_mode == "binary": + assert ( + len(normalization_statistics) == 1 + ), f"Binary normalization should only have one value, but got {normalization_statistics}" + assert normalization_statistics[0] in [ + 0, + 1, + ], f"Binary normalization should only have 0 or 1, but got {normalization_statistics[0]}" + else: + raise ValueError(f"Invalid normalization mode: {normalization_mode}") + return self + + def set_metadata(self, dataset_metadata: DatasetMetadata): + dataset_statistics = dataset_metadata.statistics + modality_metadata = dataset_metadata.modalities + + # Check that all state keys specified in apply_to have their modality_metadata + for key in self.apply_to: + split_key = key.split(".") + assert len(split_key) == 2, "State keys should have two parts: 'modality.key'" + if key not in self.modality_metadata: + modality, state_key = split_key + assert hasattr(modality_metadata, modality), f"{modality} config not found" + assert state_key in getattr( + modality_metadata, modality + ), f"{state_key} config not found" + self.modality_metadata[key] = getattr(modality_metadata, modality)[state_key] + + # Check that all state keys specified in normalization_modes have their statistics in state_statistics + for key in self.normalization_modes: + split_key = key.split(".") + assert len(split_key) == 2, "State keys should have two parts: 'modality.key'" + modality, state_key = split_key + assert hasattr(dataset_statistics, modality), f"{modality} statistics not found" + assert state_key in getattr( + dataset_statistics, modality + ), f"{state_key} statistics not found" + assert ( + len(getattr(modality_metadata, modality)[state_key].shape) == 1 + ), f"{getattr(modality_metadata, modality)[state_key].shape=}" + self.normalization_statistics[key] = getattr(dataset_statistics, modality)[ + state_key + ].model_dump() + + # Initialize the rotation transformers + for key in self.target_rotations: + # Get the original representation of the state + from_rep = self.modality_metadata[key].rotation_type + assert from_rep is not None, f"Source rotation type not found for {key}" + + # Get the target representation of the state, will raise an error if the target representation is not valid + to_rep = RotationType(self.target_rotations[key]) + + # If the original representation is not the same as the target representation, initialize the rotation transformer + if from_rep != to_rep: + self._rotation_transformers[key] = RotationTransform( + from_rep=from_rep.value, to_rep=to_rep.value + ) + + # Initialize the normalizers + for key in self.normalization_modes: + modality, state_key = key.split(".") + # If the state has a nontrivial rotation, we need to handle it more carefully + # For absolute rotations, we need to convert them to the target representation and normalize them using min_max mode, + # since we can infer the bounds by the representation + # For relative rotations, we cannot normalize them as we don't know the bounds + if key in self._rotation_transformers: + # Case 1: Absolute rotation + if self.modality_metadata[key].absolute: + # Check that the normalization mode is valid + assert ( + self.normalization_modes[key] == "min_max" + ), "Absolute rotations that are converted to other formats must be normalized using `min_max` mode" + rotation_type = RotationType(self.target_rotations[key]).value + # If the target representation is euler angles, we need to parse the convention + if rotation_type.startswith("euler_angles"): + rotation_type = "euler_angles" + # Get the statistics for the target representation + statistics = self._DEFAULT_MIN_MAX_STATISTICS[rotation_type] + # Case 2: Relative rotation + else: + raise ValueError( + f"Cannot normalize relative rotations: {key} that's converted to {self.target_rotations[key]}" + ) + # If the state is not continuous, we should not use normalization modes other than binary + elif ( + not self.modality_metadata[key].continuous + and self.normalization_modes[key] != "binary" + ): + raise ValueError( + f"{key} is not continuous, so it should be normalized using `binary` mode" + ) + # Initialize the normalizer + else: + statistics = self.normalization_statistics[key] + self._normalizers[key] = Normalizer( + mode=self.normalization_modes[key], statistics=statistics + ) + + def apply(self, data: dict[str, Any]) -> dict[str, Any]: + for key in self.apply_to: + if key not in data: + # We allow some keys to be missing in the data, and only process the keys that are present + continue + if key not in self._input_dtypes: + input_dtype = data[key].dtype + assert isinstance( + input_dtype, torch.dtype + ), f"Unexpected input dtype: {input_dtype}. Expected type: {torch.dtype}" + self._input_dtypes[key] = input_dtype + else: + assert ( + data[key].dtype == self._input_dtypes[key] + ), f"All states corresponding to the same key must be of the same dtype, input dtype: {data[key].dtype}, expected dtype: {self._input_dtypes[key]}" + # Rotate the state + state = data[key] + if key in self._rotation_transformers: + state = self._rotation_transformers[key].forward(state) + # Normalize the state + if key in self._normalizers: + state = self._normalizers[key].forward(state) + data[key] = state + return data + + def unapply(self, data: dict[str, Any]) -> dict[str, Any]: + for key in self.apply_to: + if key not in data: + continue + state = data[key] + assert isinstance( + state, torch.Tensor + ), f"Unexpected state type: {type(state)}. Expected type: {torch.Tensor}" + # Unnormalize the state + if key in self._normalizers: + state = self._normalizers[key].inverse(state) + # Change the state back to its original representation + if key in self._rotation_transformers: + state = self._rotation_transformers[key].inverse(state) + assert isinstance( + state, torch.Tensor + ), f"State should be tensor after unapplying transformations, but got {type(state)}" + # Only convert back to the original dtype if it's known, i.e. `apply` was called before + # If not, we don't know the original dtype, so we don't convert + if key in self._input_dtypes: + original_dtype = self._input_dtypes[key] + if isinstance(original_dtype, np.dtype): + state = state.numpy().astype(original_dtype) + elif isinstance(original_dtype, torch.dtype): + state = state.to(original_dtype) + else: + raise ValueError(f"Invalid input dtype: {original_dtype}") + data[key] = state + return data + + +class StateActionPerturbation(ModalityTransform): + """ + Class for state or action perturbation. + + Args: + apply_to (list[str]): The keys in the modality to load and transform. + std (float): Standard deviation of the noise to be added to the state or action. + """ + + # Configurable attributes + std: float = Field( + ..., description="Standard deviation of the noise to be added to the state or action." + ) + + def apply(self, data: dict[str, Any]) -> dict[str, Any]: + if not self.training: + # Don't perturb the data in eval mode + return data + if self.std < 0: + # If the std is negative, we don't add any noise + return data + for key in self.apply_to: + state = data[key] + assert isinstance(state, torch.Tensor) + transformed_data_min = torch.min(state) + transformed_data_max = torch.max(state) + noise = torch.randn_like(state) * self.std + state += noise + # Clip to the original range + state = torch.clamp(state, transformed_data_min, transformed_data_max) + data[key] = state + return data + + +class StateActionDropout(ModalityTransform): + """ + Class for state or action dropout. + + Args: + apply_to (list[str]): The keys in the modality to load and transform. + dropout_prob (float): Probability of dropping out a state or action. + """ + + # Configurable attributes + dropout_prob: float = Field(..., description="Probability of dropping out a state or action.") + + def apply(self, data: dict[str, Any]) -> dict[str, Any]: + if not self.training: + # Don't drop out the data in eval mode + return data + if self.dropout_prob < 0: + # If the dropout probability is negative, we don't drop out any states + return data + if self.dropout_prob > 1e-9 and random.random() < self.dropout_prob: + for key in self.apply_to: + state = data[key] + assert isinstance(state, torch.Tensor) + state = torch.zeros_like(state) + data[key] = state + return data + + +class StateActionSinCosTransform(ModalityTransform): + """ + Class for state or action sin-cos transform. + + Args: + apply_to (list[str]): The keys in the modality to load and transform. + """ + + def apply(self, data: dict[str, Any]) -> dict[str, Any]: + for key in self.apply_to: + state = data[key] + assert isinstance(state, torch.Tensor) + sin_state = torch.sin(state) + cos_state = torch.cos(state) + data[key] = torch.cat([sin_state, cos_state], dim=-1) + return data diff --git a/src/InternVLA-M1/InternVLA/dataloader/gr00t_lerobot/transform/video.py b/src/InternVLA-M1/InternVLA/dataloader/gr00t_lerobot/transform/video.py new file mode 100644 index 0000000000000000000000000000000000000000..15310f697259d7ee6ed8eac7e5abaca211b81dc9 --- /dev/null +++ b/src/InternVLA-M1/InternVLA/dataloader/gr00t_lerobot/transform/video.py @@ -0,0 +1,612 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import Any, Callable, ClassVar, Literal + +import albumentations as A +import cv2 +import numpy as np +import torch +import torchvision.transforms.v2 as T +from einops import rearrange +from pydantic import Field, PrivateAttr, field_validator +from PIL import Image + +from ..schema import DatasetMetadata +from .base import ModalityTransform + + +class VideoTransform(ModalityTransform): + # Configurable attributes + backend: str = Field( + default="torchvision", description="The backend to use for the transformations" + ) + + # Model variables + _train_transform: Callable | None = PrivateAttr(default=None) + _eval_transform: Callable | None = PrivateAttr(default=None) + _original_resolutions: dict[str, tuple[int, int]] = PrivateAttr(default_factory=dict) + + # Model constants + _INTERPOLATION_MAP: ClassVar[dict[str, dict[str, Any]]] = PrivateAttr( + { + "nearest": { + "albumentations": cv2.INTER_NEAREST, + "torchvision": T.InterpolationMode.NEAREST, + }, + "linear": { + "albumentations": cv2.INTER_LINEAR, + "torchvision": T.InterpolationMode.BILINEAR, + }, + "cubic": { + "albumentations": cv2.INTER_CUBIC, + "torchvision": T.InterpolationMode.BICUBIC, + }, + "area": { + "albumentations": cv2.INTER_AREA, + "torchvision": None, # Torchvision does not support this interpolation mode + }, + "lanczos4": { + "albumentations": cv2.INTER_LANCZOS4, # Lanczos with a 4x4 filter + "torchvision": T.InterpolationMode.LANCZOS, # Torchvision does not specify filter size, might be different from 4x4 + }, + "linear_exact": { + "albumentations": cv2.INTER_LINEAR_EXACT, + "torchvision": None, # Torchvision does not support this interpolation mode + }, + "nearest_exact": { + "albumentations": cv2.INTER_NEAREST_EXACT, + "torchvision": T.InterpolationMode.NEAREST_EXACT, + }, + "max": { + "albumentations": cv2.INTER_MAX, + "torchvision": None, + }, + } + ) + + @property + def train_transform(self) -> Callable: + assert ( + self._train_transform is not None + ), "Transform is not set. Please call set_metadata() before calling apply()." + return self._train_transform + + @train_transform.setter + def train_transform(self, value: Callable): + self._train_transform = value + + @property + def eval_transform(self) -> Callable | None: + return self._eval_transform + + @eval_transform.setter + def eval_transform(self, value: Callable | None): + self._eval_transform = value + + @property + def original_resolutions(self) -> dict[str, tuple[int, int]]: + assert ( + self._original_resolutions is not None + ), "Original resolutions are not set. Please call set_metadata() before calling apply()." + return self._original_resolutions + + @original_resolutions.setter + def original_resolutions(self, value: dict[str, tuple[int, int]]): + self._original_resolutions = value + + def check_input(self, data: dict[str, Any]): + if self.backend == "torchvision": + for key in self.apply_to: + assert isinstance(data[key], torch.Tensor), f"Video {key} is not a torch tensor" + assert data[key].ndim in [ + 4, + 5, + ], f"Expected video {key} to have 4 or 5 dimensions (T, C, H, W or T, B, C, H, W), got {data[key].ndim}" + elif self.backend == "albumentations": + for key in self.apply_to: + assert isinstance(data[key], np.ndarray), f"Video {key} is not a numpy array" + assert data[key].ndim in [ + 4, + 5, + ], f"Expected video {key} to have 4 or 5 dimensions (T, C, H, W or T, B, C, H, W), got {data[key].ndim}" + else: + raise ValueError(f"Backend {self.backend} not supported") + + def set_metadata(self, dataset_metadata: DatasetMetadata): + super().set_metadata(dataset_metadata) + self.original_resolutions = {} + for key in self.apply_to: + split_keys = key.split(".") + assert len(split_keys) == 2, f"Invalid key: {key}. Expected format: modality.key" + sub_key = split_keys[1] + if sub_key in dataset_metadata.modalities.video: + self.original_resolutions[key] = dataset_metadata.modalities.video[ + sub_key + ].resolution + else: + raise ValueError( + f"Video key {sub_key} not found in dataset metadata. Available keys: {dataset_metadata.modalities.video.keys()}" + ) + train_transform = self.get_transform(mode="train") + eval_transform = self.get_transform(mode="eval") + if self.backend == "albumentations": + self.train_transform = A.ReplayCompose(transforms=[train_transform]) # type: ignore + if eval_transform is not None: + self.eval_transform = A.ReplayCompose(transforms=[eval_transform]) # type: ignore + else: + assert train_transform is not None, "Train transform must be set" + self.train_transform = train_transform + self.eval_transform = eval_transform + + def apply(self, data: dict[str, Any]) -> dict[str, Any]: + if self.training: + transform = self.train_transform + else: + transform = self.eval_transform + if transform is None: + return data + assert ( + transform is not None + ), "Transform is not set. Please call set_metadata() before calling apply()." + try: + self.check_input(data) + except AssertionError as e: + raise ValueError( + f"Input data does not match the expected format for {self.__class__.__name__}: {e}" + ) from e + + # Concatenate views + views = [data[key] for key in self.apply_to] + num_views = len(views) + is_batched = views[0].ndim == 5 + bs = views[0].shape[0] if is_batched else 1 + if isinstance(views[0], torch.Tensor): + views = torch.cat(views, 0) + elif isinstance(views[0], np.ndarray): + views = np.concatenate(views, 0) + else: + raise ValueError(f"Unsupported view type: {type(views[0])}") + if is_batched: + views = rearrange(views, "(v b) t c h w -> (v b t) c h w", v=num_views, b=bs) + # Apply the transform + if self.backend == "torchvision": + views = transform(views) + elif self.backend == "albumentations": + assert isinstance(transform, A.ReplayCompose), "Transform must be a ReplayCompose" + first_frame = views[0] + transformed = transform(image=first_frame) + replay_data = transformed["replay"] + transformed_first_frame = transformed["image"] + + if len(views) > 1: + # Apply the same transformations to the rest of the frames + transformed_frames = [ + transform.replay(replay_data, image=frame)["image"] for frame in views[1:] + ] + # Add the first frame back + transformed_frames = [transformed_first_frame] + transformed_frames + else: + # If there is only one frame, just make a list with one frame + transformed_frames = [transformed_first_frame] + + # Delete the replay data to save memory + del replay_data + views = np.stack(transformed_frames, 0) + + else: + raise ValueError(f"Backend {self.backend} not supported") + # Split views + if is_batched: + views = rearrange(views, "(v b t) c h w -> v b t c h w", v=num_views, b=bs) + else: + views = rearrange(views, "(v t) c h w -> v t c h w", v=num_views) + for key, view in zip(self.apply_to, views): + data[key] = view + return data + + @classmethod + def _validate_interpolation(cls, interpolation: str): + if interpolation not in cls._INTERPOLATION_MAP: + raise ValueError(f"Interpolation mode {interpolation} not supported") + + def _get_interpolation(self, interpolation: str, backend: str = "torchvision"): + """ + Get the interpolation mode for the given backend. + + Args: + interpolation (str): The interpolation mode. + backend (str): The backend to use. + + Returns: + Any: The interpolation mode for the given backend. + """ + return self._INTERPOLATION_MAP[interpolation][backend] + + def get_transform(self, mode: Literal["train", "eval"] = "train") -> Callable | None: + raise NotImplementedError( + "set_transform is not implemented for VideoTransform. Please implement this function to set the transforms." + ) + + +class VideoCrop(VideoTransform): + height: int | None = Field(default=None, description="The height of the input image") + width: int | None = Field(default=None, description="The width of the input image") + scale: float = Field( + ..., + description="The scale of the crop. The crop size is (width * scale, height * scale)", + ) + + def get_transform(self, mode: Literal["train", "eval"] = "train") -> Callable: + """Get the transform for the given mode. + + Args: + mode (Literal["train", "eval"]): The mode to get the transform for. + + Returns: + Callable: If mode is "train", return a random crop transform. If mode is "eval", return a center crop transform. + """ + # 1. Check the input resolution + assert ( + len(set(self.original_resolutions.values())) == 1 + ), f"All video keys must have the same resolution, got: {self.original_resolutions}" + if self.height is None: + assert self.width is None, "Height and width must be either both provided or both None" + self.width, self.height = self.original_resolutions[self.apply_to[0]] + else: + assert ( + self.width is not None + ), "Height and width must be either both provided or both None" + # 2. Create the transform + size = (int(self.height * self.scale), int(self.width * self.scale)) + if self.backend == "torchvision": + if mode == "train": + return T.RandomCrop(size) + elif mode == "eval": + return T.CenterCrop(size) + else: + raise ValueError(f"Crop mode {mode} not supported") + elif self.backend == "albumentations": + if mode == "train": + return A.RandomCrop(height=size[0], width=size[1], p=1) + elif mode == "eval": + return A.CenterCrop(height=size[0], width=size[1], p=1) + else: + raise ValueError(f"Crop mode {mode} not supported") + else: + raise ValueError(f"Backend {self.backend} not supported") + + def check_input(self, data: dict[str, Any]): + super().check_input(data) + # Check the input resolution + for key in self.apply_to: + if self.backend == "torchvision": + height, width = data[key].shape[-2:] + elif self.backend == "albumentations": + height, width = data[key].shape[-3:-1] + else: + raise ValueError(f"Backend {self.backend} not supported") + assert ( + height == self.height and width == self.width + ), f"Video {key} has invalid shape {height, width}, expected {self.height, self.width}" + + +class VideoResize(VideoTransform): + height: int = Field(..., description="The height of the resize") + width: int = Field(..., description="The width of the resize") + interpolation: str = Field(default="linear", description="The interpolation mode") + antialias: bool = Field(default=True, description="Whether to apply antialiasing") + + @field_validator("interpolation") + def validate_interpolation(cls, v): + cls._validate_interpolation(v) + return v + + def get_transform(self, mode: Literal["train", "eval"] = "train") -> Callable: + """Get the resize transform. Same transform for both train and eval. + + Args: + mode (Literal["train", "eval"]): The mode to get the transform for. + + Returns: + Callable: The resize transform. + """ + interpolation = self._get_interpolation(self.interpolation, self.backend) + if interpolation is None: + raise ValueError( + f"Interpolation mode {self.interpolation} not supported for torchvision" + ) + if self.backend == "torchvision": + size = (self.height, self.width) + return T.Resize(size, interpolation=interpolation, antialias=self.antialias) + elif self.backend == "albumentations": + return A.Resize( + height=self.height, + width=self.width, + interpolation=interpolation, + p=1, + ) + else: + raise ValueError(f"Backend {self.backend} not supported") + + +class VideoRandomRotation(VideoTransform): + degrees: float | tuple[float, float] = Field( + ..., description="The degrees of the random rotation" + ) + interpolation: str = Field("linear", description="The interpolation mode") + + @field_validator("interpolation") + def validate_interpolation(cls, v): + cls._validate_interpolation(v) + return v + + def get_transform(self, mode: Literal["train", "eval"] = "train") -> Callable | None: + """Get the random rotation transform, only used in train mode. + + Args: + mode (Literal["train", "eval"]): The mode to get the transform for. + + Returns: + Callable | None: The random rotation transform. None for eval mode. + """ + if mode == "eval": + return None + interpolation = self._get_interpolation(self.interpolation, self.backend) + if interpolation is None: + raise ValueError( + f"Interpolation mode {self.interpolation} not supported for torchvision" + ) + if self.backend == "torchvision": + return T.RandomRotation(self.degrees, interpolation=interpolation) # type: ignore + elif self.backend == "albumentations": + return A.Rotate(limit=self.degrees, interpolation=interpolation, p=1) + else: + raise ValueError(f"Backend {self.backend} not supported") + + +class VideoHorizontalFlip(VideoTransform): + p: float = Field(..., description="The probability of the horizontal flip") + + def get_transform(self, mode: Literal["train", "eval"] = "train") -> Callable | None: + """Get the horizontal flip transform, only used in train mode. + + Args: + mode (Literal["train", "eval"]): The mode to get the transform for. + + Returns: + Callable | None: If mode is "train", return a horizontal flip transform. If mode is "eval", return None. + """ + if mode == "eval": + return None + if self.backend == "torchvision": + return T.RandomHorizontalFlip(self.p) + elif self.backend == "albumentations": + return A.HorizontalFlip(p=self.p) + else: + raise ValueError(f"Backend {self.backend} not supported") + + +class VideoGrayscale(VideoTransform): + p: float = Field(..., description="The probability of the grayscale transformation") + + def get_transform(self, mode: Literal["train", "eval"] = "train") -> Callable | None: + """Get the grayscale transform, only used in train mode. + + Args: + mode (Literal["train", "eval"]): The mode to get the transform for. + + Returns: + Callable | None: If mode is "train", return a grayscale transform. If mode is "eval", return None. + """ + if mode == "eval": + return None + if self.backend == "torchvision": + return T.RandomGrayscale(self.p) + elif self.backend == "albumentations": + return A.ToGray(p=self.p) + else: + raise ValueError(f"Backend {self.backend} not supported") + + +class VideoColorJitter(VideoTransform): + brightness: float | tuple[float, float] = Field( + ..., description="The brightness of the color jitter" + ) + contrast: float | tuple[float, float] = Field( + ..., description="The contrast of the color jitter" + ) + saturation: float | tuple[float, float] = Field( + ..., description="The saturation of the color jitter" + ) + hue: float | tuple[float, float] = Field(..., description="The hue of the color jitter") + + def get_transform(self, mode: Literal["train", "eval"] = "train") -> Callable | None: + """Get the color jitter transform, only used in train mode. + + Args: + mode (Literal["train", "eval"]): The mode to get the transform for. + + Returns: + Callable | None: If mode is "train", return a color jitter transform. If mode is "eval", return None. + """ + if mode == "eval": + return None + if self.backend == "torchvision": + return T.ColorJitter( + brightness=self.brightness, + contrast=self.contrast, + saturation=self.saturation, + hue=self.hue, + ) + elif self.backend == "albumentations": + return A.ColorJitter( + brightness=self.brightness, + contrast=self.contrast, + saturation=self.saturation, + hue=self.hue, + p=1, + ) + else: + raise ValueError(f"Backend {self.backend} not supported") + + +class VideoRandomGrayscale(VideoTransform): + p: float = Field(..., description="The probability of the grayscale transformation") + + def get_transform(self, mode: Literal["train", "eval"] = "train") -> Callable | None: + """Get the grayscale transform, only used in train mode. + + Args: + mode (Literal["train", "eval"]): The mode to get the transform for. + + Returns: + Callable | None: If mode is "train", return a grayscale transform. If mode is "eval", return None. + """ + if mode == "eval": + return None + if self.backend == "torchvision": + return T.RandomGrayscale(self.p) + elif self.backend == "albumentations": + return A.ToGray(p=self.p) + else: + raise ValueError(f"Backend {self.backend} not supported") + + +class VideoRandomPosterize(VideoTransform): + bits: int = Field(..., description="The number of bits to posterize the image") + p: float = Field(..., description="The probability of the posterize transformation") + + def get_transform(self, mode: Literal["train", "eval"] = "train") -> Callable | None: + """Get the posterize transform, only used in train mode. + + Args: + mode (Literal["train", "eval"]): The mode to get the transform for. + + Returns: + Callable | None: If mode is "train", return a posterize transform. If mode is "eval", return None. + """ + if mode == "eval": + return None + if self.backend == "torchvision": + return T.RandomPosterize(bits=self.bits, p=self.p) + elif self.backend == "albumentations": + return A.Posterize(num_bits=self.bits, p=self.p) + else: + raise ValueError(f"Backend {self.backend} not supported") + + +class VideoToTensor(VideoTransform): + def get_transform(self, mode: Literal["train", "eval"] = "train") -> Callable: + """Get the to tensor transform. Same transform for both train and eval. + + Args: + mode (Literal["train", "eval"]): The mode to get the transform for. + + Returns: + Callable: The to tensor transform. + """ + if self.backend == "torchvision": + return self.__class__.to_tensor + else: + raise ValueError(f"Backend {self.backend} not supported") + + def check_input(self, data: dict): + """Check if the input data has the correct shape. + Expected video shape: [T, H, W, C], dtype np.uint8 + """ + for key in self.apply_to: + assert key in data, f"Key {key} not found in data. Available keys: {data.keys()}" + assert data[key].ndim in [ + 4, + 5, + ], f"Video {key} must have 4 or 5 dimensions, got {data[key].ndim}" + assert ( + data[key].dtype == np.uint8 + ), f"Video {key} must have dtype uint8, got {data[key].dtype}" + input_resolution = data[key].shape[-3:-1][::-1] + if key in self.original_resolutions: + expected_resolution = self.original_resolutions[key] + else: + expected_resolution = input_resolution + assert ( + input_resolution == expected_resolution + ), f"Video {key} has invalid resolution {input_resolution}, expected {expected_resolution}. Full shape: {data[key].shape}" + + @staticmethod + def to_tensor(frames: np.ndarray) -> torch.Tensor: + """Convert numpy array to tensor efficiently. + + Args: + frames: numpy array of shape [T, H, W, C] in uint8 format + Returns: + tensor of shape [T, C, H, W] in range [0, 1] + """ + frames_tensor = torch.from_numpy(frames).to(torch.float32) / 255.0 + return frames_tensor.permute(0, 3, 1, 2) # [T, C, H, W] + + +class VideoToNumpy(VideoTransform): + def get_transform(self, mode: Literal["train", "eval"] = "train") -> Callable: + """Get the to numpy transform. Same transform for both train and eval. + + Args: + mode (Literal["train", "eval"]): The mode to get the transform for. + + Returns: + Callable: The to numpy transform. + """ + if self.backend == "torchvision": + return self.__class__.to_numpy + else: + raise ValueError(f"Backend {self.backend} not supported") + + @staticmethod + def to_numpy(frames: torch.Tensor) -> np.ndarray: + """Convert tensor back to numpy array efficiently. + + Args: + frames: tensor of shape [T, C, H, W] in range [0, 1] + Returns: + numpy array of shape [T, H, W, C] in uint8 format + """ + return (frames.permute(0, 2, 3, 1) * 255).to(torch.uint8).cpu().numpy() + +class VideoToPIL(VideoTransform): + def get_transform(self, mode: Literal["train", "eval"] = "train") -> Callable: + """Get the to PIL transform. Same transform for both train and eval. + + Args: + mode (Literal["train", "eval"]): The mode to get the transform for. + + Returns: + Callable: The to PIL transform. + """ + if self.backend == "torchvision": + return self.__class__.to_pil + else: + raise ValueError(f"Backend {self.backend} not supported") + + @staticmethod + def to_pil(frames: torch.Tensor) -> Image.Image: + """Convert tensor back to PIL Image. + + Args: + frames: tensor of shape [T, C, H, W] in range [0, 1] + Returns: + PIL Image of shape [T, H, W, C] in uint8 format + """ + # video PIL format? + return Image.fromarray((frames.permute(0, 2, 3, 1) * 255).to(torch.uint8).cpu().numpy()) \ No newline at end of file diff --git a/src/InternVLA-M1/InternVLA/dataloader/gr00t_lerobot/video.py b/src/InternVLA-M1/InternVLA/dataloader/gr00t_lerobot/video.py new file mode 100644 index 0000000000000000000000000000000000000000..eccc3597afd2530caababd9717f5499233da989e --- /dev/null +++ b/src/InternVLA-M1/InternVLA/dataloader/gr00t_lerobot/video.py @@ -0,0 +1,181 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import torch # noqa: F401 # isort: skip +import torchvision # noqa: F401 # isort: skip +import av +import cv2 +import decord # noqa: F401 +import numpy as np + + +def get_frames_by_indices( + video_path: str, + indices: list[int] | np.ndarray, + video_backend: str = "decord", + video_backend_kwargs: dict = {}, +) -> np.ndarray: + assert video_backend_kwargs == {}, "video_backend_kwargs is not supported for now" + assert video_backend == "decord", "Only decord backend is supported for now" + if video_backend == "decord": + vr = decord.VideoReader(video_path, **video_backend_kwargs) + frames = vr.get_batch(indices) + return frames.asnumpy() + elif video_backend == "opencv": + frames = [] + cap = cv2.VideoCapture(video_path, **video_backend_kwargs) + for idx in indices: + cap.set(cv2.CAP_PROP_POS_FRAMES, idx) + ret, frame = cap.read() + if not ret: + raise ValueError(f"Unable to read frame at index {idx}") + frames.append(frame) + cap.release() + frames = np.array(frames) + return frames + else: + raise NotImplementedError + + +def get_frames_by_timestamps( + video_path: str, + timestamps: list[float] | np.ndarray, + video_backend: str = "decord", + video_backend_kwargs: dict = {}, + step_indices_for_assert: list[int] | np.ndarray = None, +) -> np.ndarray: + """Get frames from a video at specified timestamps. + Args: + video_path (str): Path to the video file. + timestamps (list[int] | np.ndarray): Timestamps to retrieve frames for, in seconds. + video_backend (str, optional): Video backend to use. Defaults to "decord". + Returns: + np.ndarray: Frames at the specified timestamps. + """ + assert video_backend_kwargs == {}, "video_backend_kwargs is not supported for now" + assert video_backend == "decord", "Only decord backend is supported for now" + if video_backend == "decord": + vr = decord.VideoReader(video_path, **video_backend_kwargs) + num_frames = len(vr) + # Retrieve the timestamps for each frame in the video + frame_ts: np.ndarray = vr.get_frame_timestamp(range(num_frames)) + # Map each requested timestamp to the closest frame index + # Only take the first element of the frame_ts array which corresponds to start_seconds + indices = np.abs(frame_ts[:, :1] - timestamps).argmin(axis=0) + assert step_indices_for_assert is None or np.array_equal(indices, step_indices_for_assert), f"indices {indices} do not match step_indices_for_assert {step_indices_for_assert}" + frames = vr.get_batch(indices) + return frames.asnumpy() + elif video_backend == "opencv": + # Open the video file + cap = cv2.VideoCapture(video_path, **video_backend_kwargs) + if not cap.isOpened(): + raise ValueError(f"Unable to open video file: {video_path}") + # Retrieve the total number of frames + num_frames = int(cap.get(cv2.CAP_PROP_FRAME_COUNT)) + # Calculate timestamps for each frame + fps = cap.get(cv2.CAP_PROP_FPS) + frame_ts = np.arange(num_frames) / fps + frame_ts = frame_ts[:, np.newaxis] # Reshape to (num_frames, 1) for broadcasting + # Map each requested timestamp to the closest frame index + indices = np.abs(frame_ts - timestamps).argmin(axis=0) + frames = [] + for idx in indices: + cap.set(cv2.CAP_PROP_POS_FRAMES, idx) + ret, frame = cap.read() + if not ret: + raise ValueError(f"Unable to read frame at index {idx}") + frames.append(frame) + cap.release() + frames = np.array(frames) + return frames + elif video_backend == "torchvision_av": + # set backend + torchvision.set_video_backend("pyav") + # set a video stream reader + reader = torchvision.io.VideoReader(video_path, "video") + # set the first and last requested timestamps + # Note: previous timestamps are usually loaded, since we need to access the previous key frame + first_ts = timestamps[0] + last_ts = timestamps[-1] + # access closest key frame of the first requested frame + # Note: closest key frame timestamp is usally smaller than `first_ts` (e.g. key frame can be the first frame of the video) + # for details on what `seek` is doing see: https://pyav.basswood-io.com/docs/stable/api/container.html?highlight=inputcontainer#av.container.InputContainer.seek + reader.seek(first_ts, keyframes_only=True) + # load all frames until last requested frame + loaded_frames = [] + loaded_ts = [] + print("first_ts", first_ts) + print("last_ts", last_ts) + for frame in reader: + current_ts = frame["pts"] + print("current_ts", current_ts) + loaded_frames.append(frame["data"]) + loaded_ts.append(current_ts) + if current_ts >= last_ts: + break + if len(loaded_frames) >= len(timestamps): + break + reader.container.close() + reader = None + frames = np.array(loaded_frames) + tmp = frames.transpose(0, 2, 3, 1) + import time # save image with timestamp named image_0_timestamp.jpg + import os + os.makedirs("/hfm/boqian/tmp/test_video", exist_ok=True) + image_path = os.path.join("/hfm/boqian/tmp/test_video", f"image_{current_ts}_{time.time()}.jpg") + cv2.imwrite(image_path, tmp[0]) + return tmp + else: + raise NotImplementedError + + +def get_all_frames( + video_path: str, + video_backend: str = "decord", + video_backend_kwargs: dict = {}, + resize_size: tuple[int, int] | None = None, +) -> np.ndarray: + """Get all frames from a video. + Args: + video_path (str): Path to the video file. + video_backend (str, optional): Video backend to use. Defaults to "decord". + video_backend_kwargs (dict, optional): Keyword arguments for the video backend. + resize_size (tuple[int, int], optional): Resize size for the frames. Defaults to None. + """ + if video_backend == "decord": + vr = decord.VideoReader(video_path, **video_backend_kwargs) + frames = vr.get_batch(range(len(vr))).asnumpy() + elif video_backend == "pyav": + container = av.open(video_path) + frames = [] + for frame in container.decode(video=0): + frame = frame.to_ndarray(format="rgb24") + frames.append(frame) + frames = np.array(frames) + elif video_backend == "torchvision_av": + # set backend and reader + torchvision.set_video_backend("pyav") + reader = torchvision.io.VideoReader(video_path, "video") + frames = [] + for frame in reader: + frames.append(frame["data"]) + frames = np.array(frames) + frames = frames.transpose(0, 2, 3, 1) + else: + raise NotImplementedError(f"Video backend {video_backend} not implemented") + # resize frames if specified + if resize_size is not None: + frames = [cv2.resize(frame, resize_size) for frame in frames] + frames = np.array(frames) + return frames \ No newline at end of file diff --git a/src/InternVLA-M1/InternVLA/dataloader/lerobot_datasets.py b/src/InternVLA-M1/InternVLA/dataloader/lerobot_datasets.py new file mode 100644 index 0000000000000000000000000000000000000000..ea43a7be36142c188cdd6e2c0d7518220d6bb3ed --- /dev/null +++ b/src/InternVLA-M1/InternVLA/dataloader/lerobot_datasets.py @@ -0,0 +1,117 @@ +from pathlib import Path +from typing import Sequence +from omegaconf import OmegaConf + +from InternVLA.dataloader.gr00t_lerobot.datasets import LeRobotSingleDataset, LeRobotMixtureDataset +from InternVLA.dataloader.gr00t_lerobot.mixtures import DATASET_NAMED_MIXTURES +from InternVLA.dataloader.gr00t_lerobot.data_config import ROBOT_TYPE_CONFIG_MAP +from InternVLA.dataloader.gr00t_lerobot.embodiment_tags import ROBOT_TYPE_TO_EMBODIMENT_TAG, EmbodimentTag + +def collate_fn(batch): + return batch + +def make_LeRobotSingleDataset( + data_root_dir: Path | str, + data_name: str, + robot_type: str, + delete_pause_frame: bool = False, + preload_all: bool = False, +) -> LeRobotSingleDataset: + """ + Make a LeRobotSingleDataset object. + + :param data_root_dir: The root directory of the dataset. + :param data_name: The name of the dataset. + :param robot_type: The robot type config to use. + :param crop_obs_camera: Whether to crop the observation camera images. + :param preload_all: Whether to preload all trajectories to memory. + :return: A LeRobotSingleDataset object. + """ + + data_config = ROBOT_TYPE_CONFIG_MAP[robot_type] + modality_config = data_config.modality_config() + transforms = data_config.transform() + dataset_path = data_root_dir + if robot_type not in ROBOT_TYPE_TO_EMBODIMENT_TAG: + print(f"Warning: Robot type {robot_type} not found in ROBOT_TYPE_TO_EMBODIMENT_TAG, using {EmbodimentTag.NEW_EMBODIMENT} as default") + embodiment_tag = EmbodimentTag.NEW_EMBODIMENT + else: + embodiment_tag = ROBOT_TYPE_TO_EMBODIMENT_TAG[robot_type] + return LeRobotSingleDataset( + dataset_path=dataset_path, + modality_configs=modality_config, + transforms=transforms, + embodiment_tag=embodiment_tag, + video_backend="decord", + delete_pause_frame=delete_pause_frame, + preload_all=preload_all + ) + +def get_vla_dataset( + data_cfg: dict, + mode: str = "train", + balance_dataset_weights: bool = False, + balance_trajectory_weights: bool = False, + seed: int = 42, + delete_pause_frame: bool = True, + **kwargs: dict, +) -> LeRobotMixtureDataset: + """ + Get a LeRobotMixtureDataset object. + """ + data_root_dir = data_cfg.data_root_dir + data_mix = data_cfg.data_mix + preload_all = data_cfg.preload_all + mixture_spec = DATASET_NAMED_MIXTURES[data_mix] + included_datasets, filtered_mixture_spec = set(), [] + for d_name, d_weight, robot_type in mixture_spec: + dataset_key = (d_name, robot_type) + if dataset_key in included_datasets: + print(f"Skipping Duplicate Dataset: `{(d_name, d_weight, robot_type)}`") + continue + + included_datasets.add(dataset_key) + filtered_mixture_spec.append((d_name, d_weight, robot_type)) + + dataset_mixture = [] + for d_name, d_weight, robot_type in filtered_mixture_spec: + dataset_mixture.append((make_LeRobotSingleDataset(Path(data_root_dir), d_name, robot_type, delete_pause_frame=delete_pause_frame, preload_all=preload_all), d_weight)) + + return LeRobotMixtureDataset( + dataset_mixture, + mode=mode, + balance_dataset_weights=balance_dataset_weights, + balance_trajectory_weights=balance_trajectory_weights, + seed=seed, + **kwargs, + ) + +if __name__ == "__main__": + import debugpy + import argparse + parser = argparse.ArgumentParser() + parser.add_argument("--config_yaml", type=str, default="./InternVLA/config/training/internvla_cotrain_humanoid.yaml", help="Path to YAML config") + args, clipargs = parser.parse_known_args() + + # debugpy.listen(("0.0.0.0", 10092)) + # print("🔍 Rank 0 waiting for debugger attach on port 10092...") + # debugpy.wait_for_client() + + cfg = OmegaConf.load(args.config_yaml) + + vla_dataset_cfg = cfg.datasets.vla_data + dataset = get_vla_dataset(data_cfg=vla_dataset_cfg) + + from torch.utils.data import DataLoader + train_dataloader = DataLoader( + dataset, + batch_size=1, + num_workers=8, # For Debug + collate_fn=collate_fn, + ) + + from tqdm import tqdm + for batch in tqdm(train_dataloader, desc="Processing Batches"): + print(batch) + break + pass \ No newline at end of file diff --git a/src/InternVLA-M1/InternVLA/dataloader/qwenvl_llavajson/qwen_data_config.py b/src/InternVLA-M1/InternVLA/dataloader/qwenvl_llavajson/qwen_data_config.py new file mode 100644 index 0000000000000000000000000000000000000000..e2a11e6d2d33ec8bf81c2d8bb22ab7be98a89532 --- /dev/null +++ b/src/InternVLA-M1/InternVLA/dataloader/qwenvl_llavajson/qwen_data_config.py @@ -0,0 +1,306 @@ +import re + +from pathlib import Path + +# You can add multimodal datasets here and register a short nickname to ${data_dict}. +# The data format should follow the general multimodal VLM format, for example: +# https://github.com/QwenLM/Qwen2.5-VL/blob/main/qwen-vl-finetune/README.md + +json_root = f"./playground/Datasets/LLaVA-OneVision-Data/decoders/llava_format" +image_root = f"./playground/Datasets/LLaVA-OneVision-Data/decoders/visualData" +MAPQA_MATHV360K = { + "annotation_path": f"{json_root}/MapQA_MathV360K.json", + "data_path": f"{image_root}/", +} + +AOKVQA_CCAULDRON_LLAVA = { + "annotation_path": f"{json_root}/aokvqa_cauldron_llava_format.json", + "data_path": f"{image_root}/", +} + +SHAREGPT4V_COCO = { + "annotation_path": f"{json_root}/sharegpt4v_coco.json", + "data_path": f"{image_root}/", +} + +SHAREGPT4V_KNOWLEDGE = { + "annotation_path": f"{json_root}/sharegpt4v_knowledge.json", + "data_path": f"{image_root}/", +} + +SHAREGPT4V_LLAVA = { + "annotation_path": f"{json_root}/sharegpt4v_llava.json", + "data_path": f"{image_root}/", +} + +SHAREGPT4V_SAM = { + "annotation_path": f"{json_root}/sharegpt4v_sam.json", + "data_path": f"{image_root}/", +} + +VISUALWEBINSTRUCT_FILTERED = { + "annotation_path": f"{json_root}/VisualWebInstruct_filtered.json", + "data_path": f"{image_root}/", +} + +MAGPIE_PRO_L3_80B_MT = { + "annotation_path": f"{json_root}/magpie_pro_l3_80b_mt.json", + "data_path": "", +} + +MAGPIE_PRO_L3_80B_ST = { + "annotation_path": f"{json_root}/magpie_pro_l3_80b_st.json", + "data_path": "", +} + +VISUAL7W_CCAULDRON_LLAVA = { + "annotation_path": f"{json_root}/visual7w_cauldron_llava_format.json", + "data_path": f"{image_root}/", +} + +VISUALMRC_CCAULDRON = { + "annotation_path": f"{json_root}/visualmrc_cauldron.json", + "data_path": f"{image_root}/", +} + +VSR_CCAULDRON_LLAVA = { + "annotation_path": f"{json_root}/vsr_cauldron_llava_format.json", + "data_path": f"{image_root}/", +} + + +data_dict = { + "MapQA_MathV360K": MAPQA_MATHV360K, + "aokvqa_cauldron_llava_format": AOKVQA_CCAULDRON_LLAVA, + "sharegpt4v_coco": SHAREGPT4V_COCO, + "sharegpt4v_knowledge": SHAREGPT4V_KNOWLEDGE, + "sharegpt4v_llava": SHAREGPT4V_LLAVA, + "sharegpt4v_sam": SHAREGPT4V_SAM, + "VisualWebInstruct_filtered": VISUALWEBINSTRUCT_FILTERED, + "visual7w_cauldron_llava_format": VISUAL7W_CCAULDRON_LLAVA, + "visualmrc_cauldron": VISUALMRC_CCAULDRON, + "vsr_cauldron_llava_format": VSR_CCAULDRON_LLAVA, +} + + +def parse_sampling_rate(dataset_name): + match = re.search(r"%(\d+)$", dataset_name) + if match: + return int(match.group(1)) / 100.0 + return 1.0 + + +def data_list(dataset_names): + if dataset_names == ["all"]: + dataset_names = list(data_dict.keys()) + config_list = [] + for dataset_name in dataset_names: + sampling_rate = parse_sampling_rate(dataset_name) + dataset_name = re.sub(r"%(\d+)$", "", dataset_name) + if dataset_name in data_dict.keys(): + config = data_dict[dataset_name].copy() + config["sampling_rate"] = sampling_rate + config_list.append(config) + else: + raise ValueError(f"do not find {dataset_name}") + return config_list + + +# Kitchen +# register new Genmanip_vlm_v5 datasets + +genmanip_json_root = f"./playground/Datasets/Genmanip_vlm_v5" + +# Object-container +genmanip_object_object_root = f"{genmanip_json_root}/object_container" +GENMANIP_v5_object_container = { + "annotation_path": f"{genmanip_object_object_root}/Manipu_CoT.json", + "data_path": f"{genmanip_object_object_root}", +} + +genmanip_object_object_root = f"{genmanip_json_root}/object_object" +GENMANIP_v5_object_object = { + "annotation_path": f"{genmanip_object_object_root}/Manipu_CoT.json", + "data_path": f"{genmanip_object_object_root}", +} + +# 5*5 +genmanip_kitchen_5plus5 = f"{genmanip_json_root}/Kitchen/5x5" +GENMANIP_v5_kitchen_5plus5 = { + "annotation_path": f"{genmanip_kitchen_5plus5}/Manipu_CoT.json", + "data_path": f"{genmanip_kitchen_5plus5}", +} + +# 5*5 more +genmanip_kitchen_5plus5_more = f"{genmanip_json_root}/Kitchen/5x5_more" +GENMANIP_v5_kitchen_5plus5_more = { + "annotation_path": f"{genmanip_kitchen_5plus5_more}/Manipu_CoT.json", + "data_path": f"{genmanip_kitchen_5plus5_more}", +} + + +# Kitchen - Apple Plate +genmanip_kitchen_apple_root = f"{genmanip_json_root}/Kitchen/apple_plate" +GENMANIP_v5_kitchen_apple = { + "annotation_path": f"{genmanip_kitchen_apple_root}/Manipu_CoT.json", + "data_path": f"{genmanip_kitchen_apple_root}", +} + +# Kitchen - Banana Plate +genmanip_kitchen_banana_root = f"{genmanip_json_root}/Kitchen/banana_plate" +GENMANIP_v5_kitchen_banana = { + "annotation_path": f"{genmanip_kitchen_banana_root}/Manipu_CoT.json", + "data_path": f"{genmanip_kitchen_banana_root}", +} + +# Kitchen - Lemon Plate +genmanip_kitchen_lemon_root = f"{genmanip_json_root}/Kitchen/lemon_plate" +GENMANIP_v5_kitchen_lemon = { + "annotation_path": f"{genmanip_kitchen_lemon_root}/Manipu_CoT.json", + "data_path": f"{genmanip_kitchen_lemon_root}", +} + +# Kitchen - Sandwich Plate +genmanip_kitchen_sandwich_root = f"{genmanip_json_root}/Kitchen/sandwich_plate" +GENMANIP_v5_kitchen_sandwich = { + "annotation_path": f"{genmanip_kitchen_sandwich_root}/Manipu_CoT.json", + "data_path": f"{genmanip_kitchen_sandwich_root}", +} + +# register to data_dict +data_dict.update( + { + "GENMANIP_v5_object_object": GENMANIP_v5_object_object, + "GENMANIP_v5_object_container": GENMANIP_v5_object_container, + "GENMANIP_v5_kitchen_5plus5": GENMANIP_v5_kitchen_5plus5, + "GENMANIP_v5_kitchen_5plus5_more": GENMANIP_v5_kitchen_5plus5_more, + "GENMANIP_v5_kitchen_apple": GENMANIP_v5_kitchen_apple, + "GENMANIP_v5_kitchen_banana": GENMANIP_v5_kitchen_banana, + "GENMANIP_v5_kitchen_lemon": GENMANIP_v5_kitchen_lemon, + "GENMANIP_v5_kitchen_sandwich": GENMANIP_v5_kitchen_sandwich, + } +) + + + +# Grounding data + +llava_format_root = "/mnt/petrelfs/share/efm_p/sys2_data/coco/coco_internvl3/qwen_224_format_minp_3136_maxp_12845056" +data_root = "/mnt/petrelfs/share/efm_p/sys2_data/coco" +# Define llava_format datasets +asv2_conversation_en = { + "annotation_path": f"{llava_format_root}/asv2_conversation_en.jsonl", + "data_path": f"{data_root}", # Images may be referenced inside jsonl +} + +asv2_detailed_description_en = { + "annotation_path": f"{llava_format_root}/asv2_detailed_description_en.jsonl", + "data_path": f"{data_root}", +} + +asv2_region_captioning_en = { + "annotation_path": f"{llava_format_root}/asv2_region_captioning_en.jsonl", + "data_path": f"{data_root}", +} + +coco_internvl_longcap_en = { + "annotation_path": f"{llava_format_root}/coco_internvl_longcap_en.jsonl", + "data_path": f"{data_root}", +} + +coco_karpathy_train_567_en = { + "annotation_path": f"{llava_format_root}/coco_karpathy_train_567_en.jsonl", + "data_path": f"{data_root}", +} + +coco_negative_gpt4o_en = { + "annotation_path": f"{llava_format_root}/coco_negative_gpt4o_en.jsonl", + "data_path": f"{data_root}", +} + +coco_poetry_zh = { + "annotation_path": f"{llava_format_root}/coco_poetry_zh.jsonl", + "data_path": f"{data_root}", +} + +coco_rem_en_zh = { + "annotation_path": f"{llava_format_root}/coco_rem_en_zh.jsonl", + "data_path": f"{data_root}", +} + +cocorem_exist_yorn_en = { + "annotation_path": f"{llava_format_root}/cocorem_exist_yorn_en.jsonl", + "data_path": f"{data_root}", +} + +cocotextv2_en = { + "annotation_path": f"{llava_format_root}/cocotextv2_en.jsonl", + "data_path": f"{data_root}", +} + +cocotextv2_gpt4o_en = { + "annotation_path": f"{llava_format_root}/cocotextv2_gpt4o_en.jsonl", + "data_path": f"{data_root}", +} + +okvqa_en = {"annotation_path": f"{llava_format_root}/okvqa_en.jsonl", "data_path": f"{data_root}"} + +refcoco_grounding_aug_en = { + "annotation_path": f"{llava_format_root}/refcoco_grounding_aug_en.jsonl", + "data_path": f"{data_root}", +} + +refcoco_grounding_en = { + "annotation_path": f"{llava_format_root}/refcoco_grounding_en.jsonl", + "data_path": f"{data_root}", +} + +tallyqa_coco_en = { + "annotation_path": f"{llava_format_root}/tallyqa_coco_en.jsonl", + "data_path": f"{data_root}", +} + +toloka_grounding_aug_en = { + "annotation_path": f"{llava_format_root}/toloka_grounding_aug_en.jsonl", + "data_path": f"{data_root}", +} + +vqav2_en = { + "annotation_path": f"{llava_format_root}/vqav2_en.jsonl", + "data_path": f"{data_root}", +} + +vsr_en = { + "annotation_path": f"{llava_format_root}/vsr_en.jsonl", + "data_path": f"{data_root}", +} + +# register to data_dict +data_dict.update( + { + "asv2_conversation_en": asv2_conversation_en, + "asv2_detailed_description_en": asv2_detailed_description_en, + "asv2_region_captioning_en": asv2_region_captioning_en, + "coco_internvl_longcap_en": coco_internvl_longcap_en, + "coco_karpathy_train_567_en": coco_karpathy_train_567_en, + "coco_negative_gpt4o_en": coco_negative_gpt4o_en, + "coco_poetry_zh": coco_poetry_zh, + "coco_rem_en_zh": coco_rem_en_zh, + "cocorem_exist_yorn_en": cocorem_exist_yorn_en, + "cocotextv2_en": cocotextv2_en, + "cocotextv2_gpt4o_en": cocotextv2_gpt4o_en, + "okvqa_en": okvqa_en, + "refcoco_grounding_aug_en": refcoco_grounding_aug_en, + "refcoco_grounding_en": refcoco_grounding_en, + "tallyqa_coco_en": tallyqa_coco_en, + "toloka_grounding_aug_en": toloka_grounding_aug_en, + "vqav2_en": vqav2_en, + "vsr_en": vsr_en, + } +) + + + +if __name__ == "__main__": + print(data_list) + diff --git a/src/InternVLA-M1/InternVLA/dataloader/qwenvl_llavajson/rope2d.py b/src/InternVLA-M1/InternVLA/dataloader/qwenvl_llavajson/rope2d.py new file mode 100644 index 0000000000000000000000000000000000000000..ab4da85174dcc7da97bc27264858965628773296 --- /dev/null +++ b/src/InternVLA-M1/InternVLA/dataloader/qwenvl_llavajson/rope2d.py @@ -0,0 +1,351 @@ +import os +import copy +import json +import random +import logging +import re +import time +import math +import ast +from dataclasses import dataclass, field +from typing import Dict, Optional, Sequence, List, Tuple +from io import BytesIO +import base64 + +import numpy as np +import torch +from torch.utils.data import Dataset +from PIL import Image +from decord import VideoReader +import transformers + + +def get_rope_index_25( + spatial_merge_size: Optional[int] = 2, + input_ids: Optional[torch.LongTensor] = None, + image_grid_thw: Optional[torch.LongTensor] = None, + video_grid_thw: Optional[torch.LongTensor] = None, + second_per_grid_ts: Optional[torch.Tensor] = None, + attention_mask: Optional[torch.Tensor] = None, +) -> Tuple[torch.Tensor, torch.Tensor]: + """ + Calculate the 3D rope index based on image and video's temporal, height and width in LLM. + + Explanation: + Each embedding sequence contains vision embedding and text embedding or just contains text embedding. + + For pure text embedding sequence, the rotary position embedding has no difference with modern LLMs. + Examples: + input_ids: [T T T T T], here T is for text. + temporal position_ids: [0, 1, 2, 3, 4] + height position_ids: [0, 1, 2, 3, 4] + width position_ids: [0, 1, 2, 3, 4] + + For vision and text embedding sequence, we calculate 3D rotary position embedding for vision part + and 1D rotary position embedding for text part. + Examples: + Temporal (Time): 3 patches, representing different segments of the video in time. + Height: 2 patches, dividing each frame vertically. + Width: 2 patches, dividing each frame horizontally. + We also have some important parameters: + fps (Frames Per Second): The video's frame rate, set to 1. This means one frame is processed each second. + tokens_per_second: This is a crucial parameter. It dictates how many "time-steps" or "temporal tokens" are conceptually packed into a one-second interval of the video. In this case, we have 25 tokens per second. So each second of the video will be represented with 25 separate time points. It essentially defines the temporal granularity. + temporal_patch_size: The number of frames that compose one temporal patch. Here, it's 2 frames. + interval: The step size for the temporal position IDs, calculated as tokens_per_second * temporal_patch_size / fps. In this case, 25 * 2 / 1 = 50. This means that each temporal patch will be have a difference of 50 in the temporal position IDs. + input_ids: [V V V V V V V V V V V V T T T T T], here V is for vision. + vision temporal position_ids: [0, 0, 0, 0, 50, 50, 50, 50, 100, 100, 100, 100] + vision height position_ids: [0, 0, 1, 1, 0, 0, 1, 1, 0, 0, 1, 1] + vision width position_ids: [0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1] + text temporal position_ids: [101, 102, 103, 104, 105] + text height position_ids: [101, 102, 103, 104, 105] + text width position_ids: [101, 102, 103, 104, 105] + Here we calculate the text start position_ids as the max vision position_ids plus 1. + + Args: + input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): + Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide + it. + image_grid_thw (`torch.LongTensor` of shape `(num_images, 3)`, *optional*): + The temporal, height and width of feature shape of each image in LLM. + video_grid_thw (`torch.LongTensor` of shape `(num_videos, 3)`, *optional*): + The temporal, height and width of feature shape of each video in LLM. + second_per_grid_ts (`torch.Tensor` of shape `(num_videos)`, *optional*): + The time interval (in seconds) for each grid along the temporal dimension in the 3D position IDs. + attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*): + Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: + + - 1 for tokens that are **not masked**, + - 0 for tokens that are **masked**. + + Returns: + position_ids (`torch.LongTensor` of shape `(3, batch_size, sequence_length)`) + mrope_position_deltas (`torch.Tensor` of shape `(batch_size)`) + """ + image_token_id = 151655 + video_token_id = 151656 + vision_start_token_id = 151652 + mrope_position_deltas = [] + if input_ids is not None and (image_grid_thw is not None or video_grid_thw is not None): + total_input_ids = input_ids + if attention_mask is None: + attention_mask = torch.ones_like(total_input_ids) + position_ids = torch.ones( + 3, + input_ids.shape[0], + input_ids.shape[1], + dtype=input_ids.dtype, + device=input_ids.device, + ) + image_index, video_index = 0, 0 + attention_mask = attention_mask.to(total_input_ids.device) + for i, input_ids in enumerate(total_input_ids): + input_ids = input_ids[attention_mask[i] == 1] + image_nums, video_nums = 0, 0 + vision_start_indices = torch.argwhere(input_ids == vision_start_token_id).squeeze(1) + vision_tokens = input_ids[vision_start_indices + 1] + image_nums = (vision_tokens == image_token_id).sum() + video_nums = (vision_tokens == video_token_id).sum() + input_tokens = input_ids.tolist() + llm_pos_ids_list: list = [] + st = 0 + remain_images, remain_videos = image_nums, video_nums + for _ in range(image_nums + video_nums): + if image_token_id in input_tokens and remain_images > 0: + ed_image = input_tokens.index(image_token_id, st) + else: + ed_image = len(input_tokens) + 1 + if video_token_id in input_tokens and remain_videos > 0: + ed_video = input_tokens.index(video_token_id, st) + else: + ed_video = len(input_tokens) + 1 + if ed_image < ed_video: + t, h, w = ( + image_grid_thw[image_index][0], + image_grid_thw[image_index][1], + image_grid_thw[image_index][2], + ) + second_per_grid_t = 0 + image_index += 1 + remain_images -= 1 + ed = ed_image + + else: + t, h, w = ( + video_grid_thw[video_index][0], + video_grid_thw[video_index][1], + video_grid_thw[video_index][2], + ) + if second_per_grid_ts is not None: + second_per_grid_t = second_per_grid_ts[video_index] + else: + second_per_grid_t = 1.0 + video_index += 1 + remain_videos -= 1 + ed = ed_video + llm_grid_t, llm_grid_h, llm_grid_w = ( + t.item(), + h.item() // spatial_merge_size, + w.item() // spatial_merge_size, + ) + text_len = ed - st + + st_idx = llm_pos_ids_list[-1].max() + 1 if len(llm_pos_ids_list) > 0 else 0 + llm_pos_ids_list.append(torch.arange(text_len).view(1, -1).expand(3, -1) + st_idx) + + range_tensor = torch.arange(llm_grid_t).view(-1, 1) + expanded_range = range_tensor.expand(-1, llm_grid_h * llm_grid_w) + + time_tensor = expanded_range * second_per_grid_t * 2 + + time_tensor_long = time_tensor.long() + t_index = time_tensor_long.flatten() + + h_index = torch.arange(llm_grid_h).view(1, -1, 1).expand(llm_grid_t, -1, llm_grid_w).flatten() + w_index = torch.arange(llm_grid_w).view(1, 1, -1).expand(llm_grid_t, llm_grid_h, -1).flatten() + llm_pos_ids_list.append(torch.stack([t_index, h_index, w_index]) + text_len + st_idx) + st = ed + llm_grid_t * llm_grid_h * llm_grid_w + + if st < len(input_tokens): + st_idx = llm_pos_ids_list[-1].max() + 1 if len(llm_pos_ids_list) > 0 else 0 + text_len = len(input_tokens) - st + llm_pos_ids_list.append(torch.arange(text_len).view(1, -1).expand(3, -1) + st_idx) + + llm_positions = torch.cat(llm_pos_ids_list, dim=1).reshape(3, -1) + position_ids[..., i, attention_mask[i] == 1] = llm_positions.to(position_ids.device) + mrope_position_deltas.append(llm_positions.max() + 1 - len(total_input_ids[i])) + mrope_position_deltas = torch.tensor(mrope_position_deltas, device=input_ids.device).unsqueeze(1) + return position_ids, mrope_position_deltas + else: + if attention_mask is not None: + position_ids = attention_mask.long().cumsum(-1) - 1 + position_ids.masked_fill_(attention_mask == 0, 1) + position_ids = position_ids.unsqueeze(0).expand(3, -1, -1).to(attention_mask.device) + max_position_ids = position_ids.max(0, keepdim=False)[0].max(-1, keepdim=True)[0] + mrope_position_deltas = max_position_ids + 1 - attention_mask.shape[-1] + else: + position_ids = ( + torch.arange(input_ids.shape[1], device=input_ids.device) + .view(1, 1, -1) + .expand(3, input_ids.shape[0], -1) + ) + mrope_position_deltas = torch.zeros( + [input_ids.shape[0], 1], + device=input_ids.device, + dtype=input_ids.dtype, + ) + + return position_ids, mrope_position_deltas + + +def get_rope_index_2( + spatial_merge_size: Optional[int] = 2, + input_ids: Optional[torch.LongTensor] = None, + image_grid_thw: Optional[torch.LongTensor] = None, + video_grid_thw: Optional[torch.LongTensor] = None, + second_per_grid_ts: Optional[torch.Tensor] = None, + attention_mask: Optional[torch.Tensor] = None, +) -> Tuple[torch.Tensor, torch.Tensor]: + """ + Calculate the 3D rope index based on image and video's temporal, height and width in LLM. + + Explanation: + Each embedding sequence contains vision embedding and text embedding or just contains text embedding. + + For pure text embedding sequence, the rotary position embedding has no difference with mordern LLMs. + Examples: + input_ids: [T T T T T], here T is for text. + temporal position_ids: [0, 1, 2, 3, 4] + height position_ids: [0, 1, 2, 3, 4] + width position_ids: [0, 1, 2, 3, 4] + + For vision and text embedding sequence, we calculate 3D rotary position embedding for vision part + and 1D rotary position embeddin for text part. + Examples: + Assume we have a video input with 3 temporal patches, 2 height patches and 2 width patches. + input_ids: [V V V V V V V V V V V V T T T T T], here V is for vision. + vision temporal position_ids: [0, 0, 0, 0, 1, 1, 1, 1, 2, 2, 2, 2] + vision height position_ids: [0, 0, 1, 1, 0, 0, 1, 1, 0, 0, 1, 1] + vision width position_ids: [0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1] + text temporal position_ids: [3, 4, 5, 6, 7] + text height position_ids: [3, 4, 5, 6, 7] + text width position_ids: [3, 4, 5, 6, 7] + Here we calculate the text start position_ids as the max vision position_ids plus 1. + + Args: + input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): + Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide + it. + image_grid_thw (`torch.LongTensor` of shape `(num_images, 3)`, *optional*): + The temporal, height and width of feature shape of each image in LLM. + video_grid_thw (`torch.LongTensor` of shape `(num_videos, 3)`, *optional*): + The temporal, height and width of feature shape of each video in LLM. + attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*): + Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: + + - 1 for tokens that are **not masked**, + - 0 for tokens that are **masked**. + + Returns: + position_ids (`torch.LongTensor` of shape `(3, batch_size, sequence_length)`) + mrope_position_deltas (`torch.Tensor` of shape `(batch_size)`) + """ + image_token_id = 151655 + video_token_id = 151656 + vision_start_token_id = 151652 + mrope_position_deltas = [] + if input_ids is not None and (image_grid_thw is not None or video_grid_thw is not None): + total_input_ids = input_ids + if attention_mask is None: + attention_mask = torch.ones_like(total_input_ids) + position_ids = torch.ones( + 3, + input_ids.shape[0], + input_ids.shape[1], + dtype=input_ids.dtype, + device=input_ids.device, + ) + image_index, video_index = 0, 0 + for i, input_ids in enumerate(total_input_ids): + input_ids = input_ids[attention_mask[i] == 1] + image_nums, video_nums = 0, 0 + vision_start_indices = torch.argwhere(input_ids == vision_start_token_id).squeeze(1) + vision_tokens = input_ids[vision_start_indices + 1] + image_nums = (vision_tokens == image_token_id).sum() + video_nums = (vision_tokens == video_token_id).sum() + input_tokens = input_ids.tolist() + llm_pos_ids_list: list = [] + st = 0 + remain_images, remain_videos = image_nums, video_nums + for _ in range(image_nums + video_nums): + if image_token_id in input_tokens and remain_images > 0: + ed_image = input_tokens.index(image_token_id, st) + else: + ed_image = len(input_tokens) + 1 + if video_token_id in input_tokens and remain_videos > 0: + ed_video = input_tokens.index(video_token_id, st) + else: + ed_video = len(input_tokens) + 1 + if ed_image < ed_video: + t, h, w = ( + image_grid_thw[image_index][0], + image_grid_thw[image_index][1], + image_grid_thw[image_index][2], + ) + image_index += 1 + remain_images -= 1 + ed = ed_image + else: + t, h, w = ( + video_grid_thw[video_index][0], + video_grid_thw[video_index][1], + video_grid_thw[video_index][2], + ) + video_index += 1 + remain_videos -= 1 + ed = ed_video + llm_grid_t, llm_grid_h, llm_grid_w = ( + t.item(), + h.item() // spatial_merge_size, + w.item() // spatial_merge_size, + ) + text_len = ed - st + + st_idx = llm_pos_ids_list[-1].max() + 1 if len(llm_pos_ids_list) > 0 else 0 + llm_pos_ids_list.append(torch.arange(text_len).view(1, -1).expand(3, -1) + st_idx) + + t_index = torch.arange(llm_grid_t).view(-1, 1).expand(-1, llm_grid_h * llm_grid_w).flatten() + h_index = torch.arange(llm_grid_h).view(1, -1, 1).expand(llm_grid_t, -1, llm_grid_w).flatten() + w_index = torch.arange(llm_grid_w).view(1, 1, -1).expand(llm_grid_t, llm_grid_h, -1).flatten() + llm_pos_ids_list.append(torch.stack([t_index, h_index, w_index]) + text_len + st_idx) + st = ed + llm_grid_t * llm_grid_h * llm_grid_w + + if st < len(input_tokens): + st_idx = llm_pos_ids_list[-1].max() + 1 if len(llm_pos_ids_list) > 0 else 0 + text_len = len(input_tokens) - st + llm_pos_ids_list.append(torch.arange(text_len).view(1, -1).expand(3, -1) + st_idx) + + llm_positions = torch.cat(llm_pos_ids_list, dim=1).reshape(3, -1) + position_ids[..., i, attention_mask[i] == 1] = llm_positions.to(position_ids.device) + mrope_position_deltas.append(llm_positions.max() + 1 - len(total_input_ids[i])) + mrope_position_deltas = torch.tensor(mrope_position_deltas, device=input_ids.device).unsqueeze(1) + return position_ids, mrope_position_deltas + else: + if attention_mask is not None: + position_ids = attention_mask.long().cumsum(-1) - 1 + position_ids.masked_fill_(attention_mask == 0, 1) + position_ids = position_ids.unsqueeze(0).expand(3, -1, -1).to(attention_mask.device) + max_position_ids = position_ids.max(0, keepdim=False)[0].max(-1, keepdim=True)[0] + mrope_position_deltas = max_position_ids + 1 - attention_mask.shape[-1] + else: + position_ids = ( + torch.arange(input_ids.shape[1], device=input_ids.device) + .view(1, 1, -1) + .expand(3, input_ids.shape[0], -1) + ) + mrope_position_deltas = torch.zeros( + [input_ids.shape[0], 1], + device=input_ids.device, + dtype=input_ids.dtype, + ) + + return position_ids, mrope_position_deltas diff --git a/src/InternVLA-M1/InternVLA/dataloader/vlm_datasets.py b/src/InternVLA-M1/InternVLA/dataloader/vlm_datasets.py new file mode 100644 index 0000000000000000000000000000000000000000..933ffd6948823f07d32128712cea7471f595dc48 --- /dev/null +++ b/src/InternVLA-M1/InternVLA/dataloader/vlm_datasets.py @@ -0,0 +1,659 @@ +import os +import copy +import json +import random +import logging +import re +import time +import math +import itertools +import ast +from dataclasses import dataclass +from typing import Dict, Optional, Sequence, List, Tuple +from io import BytesIO +import base64 +from collections.abc import Sequence +from types import SimpleNamespace +import numpy as np +import torch +from torch.utils.data import Dataset +from PIL import Image +from decord import VideoReader +import transformers +from omegaconf import OmegaConf +from InternVLA.dataloader.qwenvl_llavajson.qwen_data_config import data_list +from InternVLA.dataloader.qwenvl_llavajson.rope2d import get_rope_index_25, get_rope_index_2 + +IGNORE_INDEX = -100 +IMAGE_TOKEN_INDEX = 151655 +VIDEO_TOKEN_INDEX = 151656 +DEFAULT_IMAGE_TOKEN = "\n" +DEFAULT_VIDEO_TOKEN = "