Add files using upload-large-folder tool
Browse filesThis view is limited to 50 files because it contains too many changes. See raw diff
- nix/lib/runtime.nix +74 -0
- nix/runtime-base.nix +49 -0
- nix/runtime-host-gpu.nix +101 -0
- nix/runtime.nix +14 -0
- scripts/deepspeed/zero2.json +23 -0
- scripts/deepspeed/zero3.json +29 -0
- scripts/deepspeed/zero3_offload.json +56 -0
- scripts/infer/multiturn_auto_scenario_backchannel_train_v1.json +48 -0
- scripts/infer/multiturn_auto_scenario_backchannel_val_v1.json +48 -0
- scripts/infer/multiturn_auto_scenario_emergency_stop_train_v1.json +48 -0
- scripts/infer/multiturn_auto_scenario_emergency_stop_val_v1.json +40 -0
- scripts/infer/multiturn_auto_scenario_hesitation_filled_train_v2.json +60 -0
- scripts/infer/multiturn_auto_scenario_hesitation_filled_val_v2.json +60 -0
- scripts/infer/multiturn_auto_scenario_hesitation_repair_train_v1.json +42 -0
- scripts/infer/multiturn_auto_scenario_hesitation_repair_train_v2.json +63 -0
- scripts/infer/multiturn_auto_scenario_hesitation_repair_val_v1.json +42 -0
- scripts/infer/multiturn_auto_scenario_hesitation_repair_val_v2.json +62 -0
- scripts/infer/multiturn_auto_scenario_interruption_train_v1.json +60 -0
- scripts/infer/multiturn_auto_scenario_pause_train_v1.json +41 -0
- scripts/infer/multiturn_auto_scenario_pause_train_v2.json +60 -0
- scripts/infer/multiturn_auto_scenario_safe_hold_val_v1.json +40 -0
- scripts/infer/multiturn_auto_scenario_v1.json +61 -0
- scripts/infer/multiturn_auto_scenario_val_v1.json +100 -0
- scripts/infer/multiturn_auto_suite_v2.json +92 -0
- scripts/infer/multiturn_auto_suite_v3.json +92 -0
- scripts/infer/mux_full_duplex_audio.py +180 -0
- scripts/infer/render_third_party_review.py +287 -0
- scripts/infer/render_third_party_scale.py +151 -0
- scripts/infer/run_multiturn_infer_suite.py +700 -0
- scripts/infer/seed_audio_to_motion.py +530 -0
- scripts/infer/seed_motion_teacher_reconstruct.py +504 -0
- scripts/infer/seed_text_to_motion.py +873 -0
- scripts/infer/seed_video_to_motion.py +548 -0
- scripts/infer/third_party_checkpoint_probe.py +434 -0
- scripts/infer/third_party_scale_metrics.py +172 -0
- src/InternVLA-M1/InternVLA/__init__.py +0 -0
- src/InternVLA-M1/InternVLA/config/deepseeds/deepspeed_zero2.yaml +10 -0
- src/InternVLA-M1/InternVLA/config/deepseeds/ds_config.yaml +28 -0
- src/InternVLA-M1/InternVLA/config/deepseeds/zero2.yaml +21 -0
- src/InternVLA-M1/InternVLA/config/deepseeds/zero3.json +28 -0
- src/InternVLA-M1/InternVLA/config/training/internvla_cotrain_custom.yaml +126 -0
- src/InternVLA-M1/InternVLA/config/training/internvla_cotrain_humanoid.yaml +128 -0
- src/InternVLA-M1/InternVLA/config/training/internvla_cotrain_libero.yaml +126 -0
- src/InternVLA-M1/InternVLA/config/training/internvla_cotrain_oxe.yaml +100 -0
- src/InternVLA-M1/InternVLA/config/training/internvla_cotrain_sim_demo.yaml +124 -0
- src/InternVLA-M1/InternVLA/dataloader/__init__.py +63 -0
- src/InternVLA-M1/InternVLA/dataloader/gr00t_lerobot/README.md +0 -0
- src/InternVLA-M1/InternVLA/dataloader/gr00t_lerobot/__init__.py +0 -0
- src/InternVLA-M1/InternVLA/dataloader/gr00t_lerobot/data_config.py +770 -0
- src/InternVLA-M1/InternVLA/dataloader/gr00t_lerobot/datasets.py +2118 -0
nix/lib/runtime.nix
ADDED
|
@@ -0,0 +1,74 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{ lib, pkgs }:
|
| 2 |
+
|
| 3 |
+
let
|
| 4 |
+
exportEnv = env:
|
| 5 |
+
lib.concatStringsSep "\n" (
|
| 6 |
+
lib.mapAttrsToList (name: value: "export ${name}=${lib.escapeShellArg value}") env
|
| 7 |
+
);
|
| 8 |
+
|
| 9 |
+
prependEnv = env:
|
| 10 |
+
lib.concatStringsSep "\n" (
|
| 11 |
+
lib.mapAttrsToList (
|
| 12 |
+
name: value:
|
| 13 |
+
''
|
| 14 |
+
_runtime_current_value="$(printenv ${name} || true)"
|
| 15 |
+
if [ -n "$_runtime_current_value" ]; then
|
| 16 |
+
export ${name}=${lib.escapeShellArg value}:"$_runtime_current_value"
|
| 17 |
+
else
|
| 18 |
+
export ${name}=${lib.escapeShellArg value}
|
| 19 |
+
fi
|
| 20 |
+
unset _runtime_current_value
|
| 21 |
+
''
|
| 22 |
+
) env
|
| 23 |
+
);
|
| 24 |
+
|
| 25 |
+
mergeStringAttrs = runtimes: key:
|
| 26 |
+
lib.foldl' (
|
| 27 |
+
acc: runtime:
|
| 28 |
+
let current = runtime.${key} or { };
|
| 29 |
+
in acc // lib.mapAttrs (
|
| 30 |
+
name: value:
|
| 31 |
+
if builtins.hasAttr name acc
|
| 32 |
+
then "${value}:${builtins.getAttr name acc}"
|
| 33 |
+
else value
|
| 34 |
+
) current
|
| 35 |
+
) { } runtimes;
|
| 36 |
+
|
| 37 |
+
mergeRuntimes = runtimes: {
|
| 38 |
+
packages = lib.unique (lib.concatMap (runtime: runtime.packages or [ ]) runtimes);
|
| 39 |
+
env = lib.foldl' (acc: runtime: acc // (runtime.env or { })) { } runtimes;
|
| 40 |
+
prependEnv = mergeStringAttrs runtimes "prependEnv";
|
| 41 |
+
shellHook = lib.concatStringsSep "\n\n" (
|
| 42 |
+
lib.filter (chunk: chunk != "") (map (runtime: runtime.shellHook or "") runtimes)
|
| 43 |
+
);
|
| 44 |
+
};
|
| 45 |
+
|
| 46 |
+
mkRuntimeShell = {
|
| 47 |
+
name,
|
| 48 |
+
runtimes,
|
| 49 |
+
extraPackages ? [ ],
|
| 50 |
+
extraShellHook ? "",
|
| 51 |
+
stdenv ? null,
|
| 52 |
+
}:
|
| 53 |
+
let
|
| 54 |
+
runtime = mergeRuntimes runtimes;
|
| 55 |
+
shellHook = lib.concatStringsSep "\n\n" (
|
| 56 |
+
[
|
| 57 |
+
(exportEnv (runtime.env or { }))
|
| 58 |
+
(prependEnv (runtime.prependEnv or { }))
|
| 59 |
+
(runtime.shellHook or "")
|
| 60 |
+
extraShellHook
|
| 61 |
+
''
|
| 62 |
+
echo "${name} shell ready"
|
| 63 |
+
''
|
| 64 |
+
]
|
| 65 |
+
);
|
| 66 |
+
in
|
| 67 |
+
pkgs.mkShell ({
|
| 68 |
+
packages = lib.unique (extraPackages ++ runtime.packages);
|
| 69 |
+
inherit shellHook;
|
| 70 |
+
} // lib.optionalAttrs (stdenv != null) { inherit stdenv; });
|
| 71 |
+
in
|
| 72 |
+
{
|
| 73 |
+
inherit exportEnv prependEnv mergeRuntimes mkRuntimeShell;
|
| 74 |
+
}
|
nix/runtime-base.nix
ADDED
|
@@ -0,0 +1,49 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{ pkgs }:
|
| 2 |
+
|
| 3 |
+
let
|
| 4 |
+
cuda = pkgs.cudaPackages_12_8.cudatoolkit;
|
| 5 |
+
gcc = pkgs.gcc13;
|
| 6 |
+
ffmpeg = pkgs.ffmpeg_7;
|
| 7 |
+
ffmpegLib = ffmpeg.lib;
|
| 8 |
+
caBundle = "${pkgs.cacert}/etc/ssl/certs/ca-bundle.crt";
|
| 9 |
+
in
|
| 10 |
+
{
|
| 11 |
+
packages = [
|
| 12 |
+
pkgs.cmake
|
| 13 |
+
pkgs.ninja
|
| 14 |
+
pkgs.pkg-config
|
| 15 |
+
gcc
|
| 16 |
+
pkgs.cacert
|
| 17 |
+
pkgs.linuxHeaders
|
| 18 |
+
ffmpeg
|
| 19 |
+
cuda
|
| 20 |
+
];
|
| 21 |
+
|
| 22 |
+
env = {
|
| 23 |
+
CUDA_HOME = "${cuda}";
|
| 24 |
+
CUDA_PATH = "${cuda}";
|
| 25 |
+
CUDACXX = "${cuda}/bin/nvcc";
|
| 26 |
+
CUDAHOSTCXX = "${gcc}/bin/g++";
|
| 27 |
+
CC = "${gcc}/bin/gcc";
|
| 28 |
+
CXX = "${gcc}/bin/g++";
|
| 29 |
+
CMAKE_CUDA_HOST_COMPILER = "${gcc}/bin/gcc";
|
| 30 |
+
NVCC_PREPEND_FLAGS = "-ccbin ${gcc}/bin/gcc";
|
| 31 |
+
SSL_CERT_FILE = caBundle;
|
| 32 |
+
NIX_SSL_CERT_FILE = caBundle;
|
| 33 |
+
REQUESTS_CA_BUNDLE = caBundle;
|
| 34 |
+
CURL_CA_BUNDLE = caBundle;
|
| 35 |
+
GIT_SSL_CAINFO = caBundle;
|
| 36 |
+
};
|
| 37 |
+
|
| 38 |
+
prependEnv = {
|
| 39 |
+
PATH = "${ffmpeg}/bin";
|
| 40 |
+
CPATH = "${pkgs.linuxHeaders}/include:${cuda}/include";
|
| 41 |
+
LIBRARY_PATH = "${cuda}/lib64:${cuda}/lib";
|
| 42 |
+
LD_LIBRARY_PATH = "${ffmpegLib}/lib:${cuda}/lib64:${cuda}/lib:${pkgs.stdenv.cc.cc.lib}/lib";
|
| 43 |
+
};
|
| 44 |
+
|
| 45 |
+
shellHook = ''
|
| 46 |
+
export CFLAGS="-I${pkgs.linuxHeaders}/include ''${CFLAGS:-}"
|
| 47 |
+
export CXXFLAGS="-I${pkgs.linuxHeaders}/include ''${CXXFLAGS:-}"
|
| 48 |
+
'';
|
| 49 |
+
}
|
nix/runtime-host-gpu.nix
ADDED
|
@@ -0,0 +1,101 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{ pkgs }:
|
| 2 |
+
|
| 3 |
+
{
|
| 4 |
+
shellHook = ''
|
| 5 |
+
_runtime_detect_and_prepend() {
|
| 6 |
+
_runtime_var_name="$1"
|
| 7 |
+
shift
|
| 8 |
+
_runtime_joined_paths=""
|
| 9 |
+
for _runtime_path in "$@"; do
|
| 10 |
+
if [ -e "$_runtime_path" ]; then
|
| 11 |
+
if [ -n "$_runtime_joined_paths" ]; then
|
| 12 |
+
_runtime_joined_paths="$_runtime_joined_paths:$_runtime_path"
|
| 13 |
+
else
|
| 14 |
+
_runtime_joined_paths="$_runtime_path"
|
| 15 |
+
fi
|
| 16 |
+
fi
|
| 17 |
+
done
|
| 18 |
+
|
| 19 |
+
if [ -n "$_runtime_joined_paths" ]; then
|
| 20 |
+
eval "_runtime_current_value=\''${$_runtime_var_name}"
|
| 21 |
+
if [ -n "$_runtime_current_value" ]; then
|
| 22 |
+
export "$_runtime_var_name=$_runtime_joined_paths:$_runtime_current_value"
|
| 23 |
+
else
|
| 24 |
+
export "$_runtime_var_name=$_runtime_joined_paths"
|
| 25 |
+
fi
|
| 26 |
+
unset _runtime_current_value
|
| 27 |
+
fi
|
| 28 |
+
|
| 29 |
+
unset _runtime_joined_paths
|
| 30 |
+
unset _runtime_var_name
|
| 31 |
+
unset _runtime_path
|
| 32 |
+
}
|
| 33 |
+
|
| 34 |
+
if [ -z "''${VK_ICD_FILENAMES:-}" ]; then
|
| 35 |
+
for _runtime_vk_icd in \
|
| 36 |
+
/run/opengl-driver/share/vulkan/icd.d/nvidia_icd.x86_64.json \
|
| 37 |
+
/usr/share/vulkan/icd.d/nvidia_icd.json \
|
| 38 |
+
/usr/share/vulkan/icd.d/nvidia_icd.x86_64.json \
|
| 39 |
+
/etc/vulkan/icd.d/nvidia_icd.json
|
| 40 |
+
do
|
| 41 |
+
if [ -f "$_runtime_vk_icd" ]; then
|
| 42 |
+
export VK_ICD_FILENAMES="$_runtime_vk_icd"
|
| 43 |
+
break
|
| 44 |
+
fi
|
| 45 |
+
done
|
| 46 |
+
unset _runtime_vk_icd
|
| 47 |
+
fi
|
| 48 |
+
|
| 49 |
+
if [ -z "''${__EGL_VENDOR_LIBRARY_FILENAMES:-}" ]; then
|
| 50 |
+
for _runtime_egl_vendor in \
|
| 51 |
+
/run/opengl-driver/share/glvnd/egl_vendor.d/10_nvidia.json \
|
| 52 |
+
/usr/share/glvnd/egl_vendor.d/10_nvidia.json \
|
| 53 |
+
/usr/share/glvnd/egl_vendor.d/50_nvidia.json
|
| 54 |
+
do
|
| 55 |
+
if [ -f "$_runtime_egl_vendor" ]; then
|
| 56 |
+
export __EGL_VENDOR_LIBRARY_FILENAMES="$_runtime_egl_vendor"
|
| 57 |
+
break
|
| 58 |
+
fi
|
| 59 |
+
done
|
| 60 |
+
unset _runtime_egl_vendor
|
| 61 |
+
fi
|
| 62 |
+
|
| 63 |
+
_runtime_detect_and_prepend LD_LIBRARY_PATH \
|
| 64 |
+
/run/opengl-driver/lib \
|
| 65 |
+
/run/opengl-driver-32/lib \
|
| 66 |
+
/usr/lib/x86_64-linux-gnu/nvidia/current \
|
| 67 |
+
/usr/lib/x86_64-linux-gnu/nvidia \
|
| 68 |
+
/usr/lib/nvidia \
|
| 69 |
+
/usr/lib/nvidia-570 \
|
| 70 |
+
/usr/lib/nvidia-575 \
|
| 71 |
+
/usr/lib/nvidia-580 \
|
| 72 |
+
/usr/lib/nvidia-590 \
|
| 73 |
+
/usr/lib/wsl/lib
|
| 74 |
+
|
| 75 |
+
_runtime_detect_and_prepend XDG_DATA_DIRS \
|
| 76 |
+
/run/opengl-driver/share \
|
| 77 |
+
/usr/share
|
| 78 |
+
|
| 79 |
+
if [ -z "''${TRITON_LIBCUDA_PATH:-}" ]; then
|
| 80 |
+
for _runtime_triton_cuda in \
|
| 81 |
+
/run/opengl-driver/lib \
|
| 82 |
+
/usr/lib/x86_64-linux-gnu \
|
| 83 |
+
/usr/lib/x86_64-linux-gnu/nvidia/current \
|
| 84 |
+
/usr/lib/x86_64-linux-gnu/nvidia \
|
| 85 |
+
/usr/lib/nvidia-590 \
|
| 86 |
+
/usr/lib/nvidia-580 \
|
| 87 |
+
/usr/lib/nvidia-575 \
|
| 88 |
+
/usr/lib/nvidia-570 \
|
| 89 |
+
/usr/lib/wsl/lib
|
| 90 |
+
do
|
| 91 |
+
if [ -d "$_runtime_triton_cuda" ]; then
|
| 92 |
+
export TRITON_LIBCUDA_PATH="$_runtime_triton_cuda"
|
| 93 |
+
break
|
| 94 |
+
fi
|
| 95 |
+
done
|
| 96 |
+
unset _runtime_triton_cuda
|
| 97 |
+
fi
|
| 98 |
+
|
| 99 |
+
unset -f _runtime_detect_and_prepend
|
| 100 |
+
'';
|
| 101 |
+
}
|
nix/runtime.nix
ADDED
|
@@ -0,0 +1,14 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{ pkgs }:
|
| 2 |
+
|
| 3 |
+
let
|
| 4 |
+
runtimeLib = import ./lib/runtime.nix {
|
| 5 |
+
lib = pkgs.lib;
|
| 6 |
+
inherit pkgs;
|
| 7 |
+
};
|
| 8 |
+
baseRuntime = import ./runtime-base.nix { inherit pkgs; };
|
| 9 |
+
hostGpuRuntime = import ./runtime-host-gpu.nix { inherit pkgs; };
|
| 10 |
+
in
|
| 11 |
+
runtimeLib.mergeRuntimes [
|
| 12 |
+
baseRuntime
|
| 13 |
+
hostGpuRuntime
|
| 14 |
+
]
|
scripts/deepspeed/zero2.json
ADDED
|
@@ -0,0 +1,23 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"fp16": {
|
| 3 |
+
"enabled": "auto",
|
| 4 |
+
"loss_scale": 0,
|
| 5 |
+
"loss_scale_window": 1000,
|
| 6 |
+
"initial_scale_power": 16,
|
| 7 |
+
"hysteresis": 2,
|
| 8 |
+
"min_loss_scale": 1
|
| 9 |
+
},
|
| 10 |
+
"bf16": {
|
| 11 |
+
"enabled": "auto"
|
| 12 |
+
},
|
| 13 |
+
"train_micro_batch_size_per_gpu": "auto",
|
| 14 |
+
"train_batch_size": "auto",
|
| 15 |
+
"gradient_accumulation_steps": "auto",
|
| 16 |
+
"zero_optimization": {
|
| 17 |
+
"stage": 2,
|
| 18 |
+
"overlap_comm": true,
|
| 19 |
+
"contiguous_gradients": true,
|
| 20 |
+
"sub_group_size": 1e9,
|
| 21 |
+
"reduce_bucket_size": "auto"
|
| 22 |
+
}
|
| 23 |
+
}
|
scripts/deepspeed/zero3.json
ADDED
|
@@ -0,0 +1,29 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"fp16": {
|
| 3 |
+
"enabled": "auto",
|
| 4 |
+
"loss_scale": 0,
|
| 5 |
+
"loss_scale_window": 1000,
|
| 6 |
+
"initial_scale_power": 16,
|
| 7 |
+
"hysteresis": 2,
|
| 8 |
+
"min_loss_scale": 1
|
| 9 |
+
},
|
| 10 |
+
"bf16": {
|
| 11 |
+
"enabled": "auto"
|
| 12 |
+
},
|
| 13 |
+
"train_micro_batch_size_per_gpu": "auto",
|
| 14 |
+
"train_batch_size": "auto",
|
| 15 |
+
"gradient_accumulation_steps": "auto",
|
| 16 |
+
"gradient_clipping": 1.0,
|
| 17 |
+
"zero_optimization": {
|
| 18 |
+
"stage": 3,
|
| 19 |
+
"overlap_comm": true,
|
| 20 |
+
"contiguous_gradients": true,
|
| 21 |
+
"sub_group_size": 1e9,
|
| 22 |
+
"reduce_bucket_size": "auto",
|
| 23 |
+
"stage3_prefetch_bucket_size": "auto",
|
| 24 |
+
"stage3_param_persistence_threshold": "auto",
|
| 25 |
+
"stage3_max_live_parameters": 1e9,
|
| 26 |
+
"stage3_max_reuse_distance": 1e9,
|
| 27 |
+
"stage3_gather_16bit_weights_on_model_save": true
|
| 28 |
+
}
|
| 29 |
+
}
|
scripts/deepspeed/zero3_offload.json
ADDED
|
@@ -0,0 +1,56 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"fp16": {
|
| 3 |
+
"enabled": "auto",
|
| 4 |
+
"loss_scale": 0,
|
| 5 |
+
"loss_scale_window": 1000,
|
| 6 |
+
"initial_scale_power": 16,
|
| 7 |
+
"hysteresis": 2,
|
| 8 |
+
"min_loss_scale": 1
|
| 9 |
+
},
|
| 10 |
+
"bf16": {
|
| 11 |
+
"enabled": "auto"
|
| 12 |
+
},
|
| 13 |
+
"optimizer": {
|
| 14 |
+
"type": "AdamW",
|
| 15 |
+
"params": {
|
| 16 |
+
"lr": "auto",
|
| 17 |
+
"betas": "auto",
|
| 18 |
+
"eps": "auto",
|
| 19 |
+
"weight_decay": "auto"
|
| 20 |
+
}
|
| 21 |
+
},
|
| 22 |
+
"scheduler": {
|
| 23 |
+
"type": "WarmupLR",
|
| 24 |
+
"params": {
|
| 25 |
+
"warmup_min_lr": "auto",
|
| 26 |
+
"warmup_max_lr": "auto",
|
| 27 |
+
"warmup_num_steps": "auto"
|
| 28 |
+
}
|
| 29 |
+
},
|
| 30 |
+
"zero_optimization": {
|
| 31 |
+
"stage": 3,
|
| 32 |
+
"offload_optimizer": {
|
| 33 |
+
"device": "cpu",
|
| 34 |
+
"pin_memory": true
|
| 35 |
+
},
|
| 36 |
+
"offload_param": {
|
| 37 |
+
"device": "cpu",
|
| 38 |
+
"pin_memory": true
|
| 39 |
+
},
|
| 40 |
+
"overlap_comm": true,
|
| 41 |
+
"contiguous_gradients": true,
|
| 42 |
+
"sub_group_size": 1e9,
|
| 43 |
+
"reduce_bucket_size": "auto",
|
| 44 |
+
"stage3_prefetch_bucket_size": "auto",
|
| 45 |
+
"stage3_param_persistence_threshold": "auto",
|
| 46 |
+
"stage3_max_live_parameters": 1e9,
|
| 47 |
+
"stage3_max_reuse_distance": 1e9,
|
| 48 |
+
"gather_16bit_weights_on_model_save": true
|
| 49 |
+
},
|
| 50 |
+
"gradient_accumulation_steps": "auto",
|
| 51 |
+
"gradient_clipping": "auto",
|
| 52 |
+
"train_batch_size": "auto",
|
| 53 |
+
"train_micro_batch_size_per_gpu": "auto",
|
| 54 |
+
"steps_per_print": 1e5,
|
| 55 |
+
"wall_clock_breakdown": false
|
| 56 |
+
}
|
scripts/infer/multiturn_auto_scenario_backchannel_train_v1.json
ADDED
|
@@ -0,0 +1,48 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"format": "hfd_multiturn_auto_infer_scenario_v1",
|
| 3 |
+
"name": "backchannel_grab_mmhmm_wave_zh",
|
| 4 |
+
"interaction_mode": "backchannel",
|
| 5 |
+
"data_split": "train",
|
| 6 |
+
"split_seed": 42,
|
| 7 |
+
"val_ratio": 0.1,
|
| 8 |
+
"split_source_namespace": "seed",
|
| 9 |
+
"description": "第一条动作执行中插入同音色的训练集短反馈“嗯嗯”,动作必须继续;随后再给一条真正的新指令。",
|
| 10 |
+
"required_semantics_version": 22,
|
| 11 |
+
"initial_idle_steps": 5,
|
| 12 |
+
"initial_idle_seconds": 2.0,
|
| 13 |
+
"motion_frames_per_turn": 450,
|
| 14 |
+
"motion_seconds_per_turn": 9.0,
|
| 15 |
+
"inter_turn_background_steps_by_turn": [10, 15],
|
| 16 |
+
"inter_turn_background_seconds_by_turn": [4.0, 6.0],
|
| 17 |
+
"terminal_background_steps": 5,
|
| 18 |
+
"terminal_background_seconds": 2.0,
|
| 19 |
+
"turns": [
|
| 20 |
+
{
|
| 21 |
+
"turn_index": 1,
|
| 22 |
+
"source_motion_filename": "grab_walk_ff_180_001__A548",
|
| 23 |
+
"audio_path": "/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/qwen3-tts/seed_audio/000006/female_01_zh.wav",
|
| 24 |
+
"spoken_command": "向前走并抓取某物",
|
| 25 |
+
"ideal_motion": "听完后向前走并抓取,动作持续执行。"
|
| 26 |
+
},
|
| 27 |
+
{
|
| 28 |
+
"turn_index": 2,
|
| 29 |
+
"backchannel_intent": "continuer",
|
| 30 |
+
"audio_path": "/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/videogen/hfd_backchannel_commands_v2_short/audio/train/bc_cont_01/female_01_zh.wav",
|
| 31 |
+
"spoken_command": "嗯嗯",
|
| 32 |
+
"ideal_motion": "这是短反馈而不是新指令;控制持续输出 </motion>,抓取动作的相位、RTC 和 future progress 均连续,不进入 listen 或 idle。"
|
| 33 |
+
},
|
| 34 |
+
{
|
| 35 |
+
"turn_index": 3,
|
| 36 |
+
"source_motion_filename": "wave_two_hands_R_001__A533",
|
| 37 |
+
"audio_path": "/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/qwen3-tts/seed_audio/000155/female_01_zh.wav",
|
| 38 |
+
"spoken_command": "双手在身前挥动",
|
| 39 |
+
"ideal_motion": "这是真正的新指令,模型才应进入 listen,随后从当前状态执行双手挥动。"
|
| 40 |
+
}
|
| 41 |
+
],
|
| 42 |
+
"ideal_control_timeline": [
|
| 43 |
+
"第一条动作指令遵循 idle→listen→motion",
|
| 44 |
+
"动作中听到“嗯嗯”仍保持 motion,不重置动作相位",
|
| 45 |
+
"后续真正的新指令到来后才从 motion 切到 listen,再进入新 motion",
|
| 46 |
+
"末尾纯背景回到 idle"
|
| 47 |
+
]
|
| 48 |
+
}
|
scripts/infer/multiturn_auto_scenario_backchannel_val_v1.json
ADDED
|
@@ -0,0 +1,48 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"format": "hfd_multiturn_auto_infer_scenario_v1",
|
| 3 |
+
"name": "backchannel_jog_indeed_pick_en",
|
| 4 |
+
"interaction_mode": "backchannel",
|
| 5 |
+
"data_split": "val",
|
| 6 |
+
"split_seed": 42,
|
| 7 |
+
"val_ratio": 0.1,
|
| 8 |
+
"split_source_namespace": "seed",
|
| 9 |
+
"description": "未见过的验证动作与验证短反馈组合;检查 backchannel 的语义泛化,而不是记住固定词。",
|
| 10 |
+
"required_semantics_version": 22,
|
| 11 |
+
"initial_idle_steps": 4,
|
| 12 |
+
"initial_idle_seconds": 1.6,
|
| 13 |
+
"motion_frames_per_turn": 450,
|
| 14 |
+
"motion_seconds_per_turn": 9.0,
|
| 15 |
+
"inter_turn_background_steps_by_turn": [9, 16],
|
| 16 |
+
"inter_turn_background_seconds_by_turn": [3.6, 6.4],
|
| 17 |
+
"terminal_background_steps": 6,
|
| 18 |
+
"terminal_background_seconds": 2.4,
|
| 19 |
+
"turns": [
|
| 20 |
+
{
|
| 21 |
+
"turn_index": 1,
|
| 22 |
+
"source_motion_filename": "jog_ff_start_180_R_003__A068",
|
| 23 |
+
"audio_path": "/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/qwen3-tts/seed_audio/000175/female_02_en.wav",
|
| 24 |
+
"spoken_command": "start jog forward",
|
| 25 |
+
"ideal_motion": "听完后向前慢跑,并保持连续闭环状态。"
|
| 26 |
+
},
|
| 27 |
+
{
|
| 28 |
+
"turn_index": 2,
|
| 29 |
+
"backchannel_intent": "continuer",
|
| 30 |
+
"audio_path": "/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/videogen/hfd_backchannel_commands_v2_short/audio/val/bc_cont_11/female_02_en.wav",
|
| 31 |
+
"spoken_command": "Indeed.",
|
| 32 |
+
"ideal_motion": "验证集短反馈不应抢占控制权;慢跑动作、RTC、进度和 Qwen KV 均连续。"
|
| 33 |
+
},
|
| 34 |
+
{
|
| 35 |
+
"turn_index": 3,
|
| 36 |
+
"source_motion_filename": "medium_big_light_two_hands_right_side_low_to_right_side_high_R_001__A529",
|
| 37 |
+
"audio_path": "/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/qwen3-tts/seed_audio/000227/female_02_en.wav",
|
| 38 |
+
"spoken_command": "picks up medium big light object with two hands from low right side and puts it away to the right side high",
|
| 39 |
+
"ideal_motion": "真正的新指令到来后才切入 listen,随后平滑执行右侧双手取放动作。"
|
| 40 |
+
}
|
| 41 |
+
],
|
| 42 |
+
"ideal_control_timeline": [
|
| 43 |
+
"初始背景输出 idle,第一条动作指令进入 listen 后执行 motion",
|
| 44 |
+
"未见过的短反馈期间持续输出 motion,不出现假 interruption",
|
| 45 |
+
"下一条真实指令出现后才切换 listen 和新 motion",
|
| 46 |
+
"最后在背景中回到 idle"
|
| 47 |
+
]
|
| 48 |
+
}
|
scripts/infer/multiturn_auto_scenario_emergency_stop_train_v1.json
ADDED
|
@@ -0,0 +1,48 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"format": "hfd_multiturn_auto_infer_scenario_v1",
|
| 3 |
+
"name": "stop_train_grab_stop_wave_zh",
|
| 4 |
+
"interaction_mode": "emergency-stop",
|
| 5 |
+
"data_split": "train",
|
| 6 |
+
"split_seed": 42,
|
| 7 |
+
"val_ratio": 0.1,
|
| 8 |
+
"split_source_namespace": "seed",
|
| 9 |
+
"description": "训练集抓取行走执行中收到中文急停,停住并保持两秒背景后再接挥手指令,检查控制后的恢复交互。",
|
| 10 |
+
"required_semantics_version": 21,
|
| 11 |
+
"initial_idle_steps": 5,
|
| 12 |
+
"initial_idle_seconds": 2.0,
|
| 13 |
+
"motion_frames_per_turn": 450,
|
| 14 |
+
"motion_seconds_per_turn": 9.0,
|
| 15 |
+
"inter_turn_background_steps_by_turn": [8, 5],
|
| 16 |
+
"inter_turn_background_seconds_by_turn": [3.2, 2.0],
|
| 17 |
+
"terminal_background_steps": 5,
|
| 18 |
+
"terminal_background_seconds": 2.0,
|
| 19 |
+
"turns": [
|
| 20 |
+
{
|
| 21 |
+
"turn_index": 1,
|
| 22 |
+
"source_motion_filename": "grab_walk_ff_180_001__A548",
|
| 23 |
+
"audio_path": "/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/qwen3-tts/seed_audio/000006/female_01_zh.wav",
|
| 24 |
+
"spoken_command": "向前走并抓取某物",
|
| 25 |
+
"ideal_motion": "听完后向前行走并开始抓取。"
|
| 26 |
+
},
|
| 27 |
+
{
|
| 28 |
+
"turn_index": 2,
|
| 29 |
+
"control_intent": "emergency-stop",
|
| 30 |
+
"audio_path": "/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/videogen/hfd_control_commands_v1/audio/emergency-stop/stop_02/female_01_zh.wav",
|
| 31 |
+
"spoken_command": "马上停下来",
|
| 32 |
+
"ideal_motion": "完整听到急停后直接从 motion 切到 idle,平滑停住,不出现 listen。"
|
| 33 |
+
},
|
| 34 |
+
{
|
| 35 |
+
"turn_index": 3,
|
| 36 |
+
"source_motion_filename": "wave_two_hands_R_001__A533",
|
| 37 |
+
"audio_path": "/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/qwen3-tts/seed_audio/000155/female_01_zh.wav",
|
| 38 |
+
"spoken_command": "双手在身前挥动",
|
| 39 |
+
"ideal_motion": "急停后保持约两秒,再正常进入 listen 并执行双手挥动。"
|
| 40 |
+
}
|
| 41 |
+
],
|
| 42 |
+
"ideal_control_timeline": [
|
| 43 |
+
"初始背景为 </idle>,第一条动作指令期间为 </listen>",
|
| 44 |
+
"执行 </motion> 约 3.2 秒后收到急停语音",
|
| 45 |
+
"急停语音未听完前仍是 </motion>,听完后直接转 </idle>",
|
| 46 |
+
"保持约两秒连续背景后接收新指令,重新进入 </listen> → </motion>"
|
| 47 |
+
]
|
| 48 |
+
}
|
scripts/infer/multiturn_auto_scenario_emergency_stop_val_v1.json
ADDED
|
@@ -0,0 +1,40 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"format": "hfd_multiturn_auto_infer_scenario_v1",
|
| 3 |
+
"name": "stop_val_jog_en",
|
| 4 |
+
"interaction_mode": "emergency-stop",
|
| 5 |
+
"data_split": "val",
|
| 6 |
+
"split_seed": 42,
|
| 7 |
+
"val_ratio": 0.1,
|
| 8 |
+
"split_source_namespace": "seed",
|
| 9 |
+
"description": "未见过的慢跑 motion 执行中收到英文急停,检查特殊指令的泛化。",
|
| 10 |
+
"required_semantics_version": 21,
|
| 11 |
+
"initial_idle_steps": 4,
|
| 12 |
+
"initial_idle_seconds": 1.6,
|
| 13 |
+
"motion_frames_per_turn": 450,
|
| 14 |
+
"motion_seconds_per_turn": 9.0,
|
| 15 |
+
"inter_turn_background_steps_by_turn": [9],
|
| 16 |
+
"inter_turn_background_seconds_by_turn": [3.6],
|
| 17 |
+
"terminal_background_steps": 6,
|
| 18 |
+
"terminal_background_seconds": 2.4,
|
| 19 |
+
"turns": [
|
| 20 |
+
{
|
| 21 |
+
"turn_index": 1,
|
| 22 |
+
"source_motion_filename": "jog_ff_start_180_R_003__A068",
|
| 23 |
+
"audio_path": "/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/qwen3-tts/seed_audio/000175/female_02_en.wav",
|
| 24 |
+
"spoken_command": "start jog forward",
|
| 25 |
+
"ideal_motion": "开始向前慢跑。"
|
| 26 |
+
},
|
| 27 |
+
{
|
| 28 |
+
"turn_index": 2,
|
| 29 |
+
"control_intent": "emergency-stop",
|
| 30 |
+
"audio_path": "/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/videogen/hfd_control_commands_v1/audio/emergency-stop/stop_02/female_02_en.wav",
|
| 31 |
+
"spoken_command": "Stop now.",
|
| 32 |
+
"ideal_motion": "完整听到 stop 后直接平滑停住,不进入 listen。"
|
| 33 |
+
}
|
| 34 |
+
],
|
| 35 |
+
"ideal_control_timeline": [
|
| 36 |
+
"idle -> listen -> motion",
|
| 37 |
+
"急停语音在 motion 中到来",
|
| 38 |
+
"听清后 motion -> idle,中间没有 listen"
|
| 39 |
+
]
|
| 40 |
+
}
|
scripts/infer/multiturn_auto_scenario_hesitation_filled_train_v2.json
ADDED
|
@@ -0,0 +1,60 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"format": "hfd_multiturn_auto_infer_scenario_v1",
|
| 3 |
+
"name": "filled_wave_then_left_zh",
|
| 4 |
+
"interaction_mode": "hesitation",
|
| 5 |
+
"interaction_subtype": "filled-pause",
|
| 6 |
+
"data_split": "train",
|
| 7 |
+
"split_seed": 42,
|
| 8 |
+
"val_ratio": 0.1,
|
| 9 |
+
"split_source_namespace": "seed",
|
| 10 |
+
"description": "训练集中文指令中插入同音色、自然拖长的“呃……”,实际不确定区间为 2.4 秒,最终意图不变。",
|
| 11 |
+
"required_semantics_version": 24,
|
| 12 |
+
"initial_idle_steps": 5,
|
| 13 |
+
"initial_idle_seconds": 2.0,
|
| 14 |
+
"motion_frames_per_turn": 450,
|
| 15 |
+
"motion_seconds_per_turn": 9.0,
|
| 16 |
+
"inter_turn_background_steps_by_turn": [
|
| 17 |
+
25
|
| 18 |
+
],
|
| 19 |
+
"inter_turn_background_seconds_by_turn": [
|
| 20 |
+
10.0
|
| 21 |
+
],
|
| 22 |
+
"terminal_background_steps": 5,
|
| 23 |
+
"terminal_background_seconds": 2.0,
|
| 24 |
+
"disfluency_event": {
|
| 25 |
+
"turn_index": 0,
|
| 26 |
+
"subtype": "filled-pause",
|
| 27 |
+
"gap_steps": 6,
|
| 28 |
+
"gap_seconds": 2.4,
|
| 29 |
+
"marker_id": "hes_fill_zh_02"
|
| 30 |
+
},
|
| 31 |
+
"turns": [
|
| 32 |
+
{
|
| 33 |
+
"turn_index": 1,
|
| 34 |
+
"disfluency_subtype": "filled-pause",
|
| 35 |
+
"source_motion_filename": "wave_two_hands_R_001__A533",
|
| 36 |
+
"audio_path": "/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/videogen/hfd_hesitation_pause_v2/audio/review_examples/train/filled_wave_zh_female_01_zh.wav",
|
| 37 |
+
"spoken_command": "双手在身前……呃……挥动",
|
| 38 |
+
"final_instruction": "双手在身前挥动",
|
| 39 |
+
"ideal_motion": "听到呃时继续 listen,不提前抬手;完整指令结束后再完成双手挥动。"
|
| 40 |
+
},
|
| 41 |
+
{
|
| 42 |
+
"turn_index": 2,
|
| 43 |
+
"source_motion_filename": "neutral_walk_ff_270_R_001__A534_M",
|
| 44 |
+
"audio_path": "/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/qwen3-tts/seed_audio/000120/female_01_zh.wav",
|
| 45 |
+
"spoken_command": "向左走路",
|
| 46 |
+
"ideal_motion": "随后相对当前朝向向左走。"
|
| 47 |
+
}
|
| 48 |
+
],
|
| 49 |
+
"ideal_control_timeline": [
|
| 50 |
+
"背景 idle",
|
| 51 |
+
"前半句、呃和后半句均为 listen",
|
| 52 |
+
"只有最终意图完整后才进入 motion",
|
| 53 |
+
"第二轮正常执行"
|
| 54 |
+
],
|
| 55 |
+
"continuity_requirements": [
|
| 56 |
+
"同一女性音色和语言",
|
| 57 |
+
"同一 Qwen KV 与连续背景",
|
| 58 |
+
"RTC 和动作相位不在犹豫处重置"
|
| 59 |
+
]
|
| 60 |
+
}
|
scripts/infer/multiturn_auto_scenario_hesitation_filled_val_v2.json
ADDED
|
@@ -0,0 +1,60 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"format": "hfd_multiturn_auto_infer_scenario_v1",
|
| 3 |
+
"name": "filled_pick_then_jog_en",
|
| 4 |
+
"interaction_mode": "hesitation",
|
| 5 |
+
"interaction_subtype": "filled-pause",
|
| 6 |
+
"data_split": "val",
|
| 7 |
+
"split_seed": 42,
|
| 8 |
+
"val_ratio": 0.1,
|
| 9 |
+
"split_source_namespace": "seed",
|
| 10 |
+
"description": "验证集英文长指令中插入未用于训练的同音色、自然拖长的 Hmm,检查长语义在 KV 中继续累积。",
|
| 11 |
+
"required_semantics_version": 24,
|
| 12 |
+
"initial_idle_steps": 4,
|
| 13 |
+
"initial_idle_seconds": 1.6,
|
| 14 |
+
"motion_frames_per_turn": 450,
|
| 15 |
+
"motion_seconds_per_turn": 9.0,
|
| 16 |
+
"inter_turn_background_steps_by_turn": [
|
| 17 |
+
25
|
| 18 |
+
],
|
| 19 |
+
"inter_turn_background_seconds_by_turn": [
|
| 20 |
+
10.0
|
| 21 |
+
],
|
| 22 |
+
"terminal_background_steps": 6,
|
| 23 |
+
"terminal_background_seconds": 2.4,
|
| 24 |
+
"disfluency_event": {
|
| 25 |
+
"turn_index": 0,
|
| 26 |
+
"subtype": "filled-pause",
|
| 27 |
+
"gap_steps": 5,
|
| 28 |
+
"gap_seconds": 2.0,
|
| 29 |
+
"marker_id": "hes_fill_en_11"
|
| 30 |
+
},
|
| 31 |
+
"turns": [
|
| 32 |
+
{
|
| 33 |
+
"turn_index": 1,
|
| 34 |
+
"disfluency_subtype": "filled-pause",
|
| 35 |
+
"source_motion_filename": "medium_big_light_two_hands_right_side_low_to_right_side_high_R_001__A529",
|
| 36 |
+
"audio_path": "/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/videogen/hfd_hesitation_pause_v2/audio/review_examples/val/filled_pick_en_female_02_en.wav",
|
| 37 |
+
"spoken_command": "picks up ... from low ... Hmm ... right side and puts it away to the right side high",
|
| 38 |
+
"final_instruction": "picks up medium big light object with two hands from low right side and puts it away to the right side high",
|
| 39 |
+
"ideal_motion": "Hmm 前后都保持 listen,完整结束后才执行右下到右上的双手取放。"
|
| 40 |
+
},
|
| 41 |
+
{
|
| 42 |
+
"turn_index": 2,
|
| 43 |
+
"source_motion_filename": "jog_ff_start_180_R_003__A068",
|
| 44 |
+
"audio_path": "/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/qwen3-tts/seed_audio/000175/female_02_en.wav",
|
| 45 |
+
"spoken_command": "start jog forward",
|
| 46 |
+
"ideal_motion": "随后从当前状态向前慢跑。"
|
| 47 |
+
}
|
| 48 |
+
],
|
| 49 |
+
"ideal_control_timeline": [
|
| 50 |
+
"初始 idle",
|
| 51 |
+
"整条含 Hmm 的未完成指令保持 listen",
|
| 52 |
+
"完整语义后进入 motion",
|
| 53 |
+
"第二轮正常执行"
|
| 54 |
+
],
|
| 55 |
+
"continuity_requirements": [
|
| 56 |
+
"验证 marker 与训练 marker 含义拆分",
|
| 57 |
+
"同一音色、KV、背景和 RTC",
|
| 58 |
+
"两条 motion source 均属于 val"
|
| 59 |
+
]
|
| 60 |
+
}
|
scripts/infer/multiturn_auto_scenario_hesitation_repair_train_v1.json
ADDED
|
@@ -0,0 +1,42 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"format": "hfd_multiturn_auto_infer_scenario_v1",
|
| 3 |
+
"name": "repair_grab_to_left_then_wave_zh",
|
| 4 |
+
"interaction_mode": "hesitation",
|
| 5 |
+
"interaction_subtype": "self-repair",
|
| 6 |
+
"data_split": "train",
|
| 7 |
+
"split_seed": 42,
|
| 8 |
+
"val_ratio": 0.1,
|
| 9 |
+
"split_source_namespace": "seed",
|
| 10 |
+
"description": "先说出抓取指令的不完整前缀,再用同音色明确改口为完整向左走;只能执行最终指令。",
|
| 11 |
+
"required_semantics_version": 23,
|
| 12 |
+
"initial_idle_steps": 5,
|
| 13 |
+
"initial_idle_seconds": 2.0,
|
| 14 |
+
"motion_frames_per_turn": 450,
|
| 15 |
+
"motion_seconds_per_turn": 9.0,
|
| 16 |
+
"inter_turn_background_steps_by_turn": [25],
|
| 17 |
+
"inter_turn_background_seconds_by_turn": [10.0],
|
| 18 |
+
"terminal_background_steps": 5,
|
| 19 |
+
"terminal_background_seconds": 2.0,
|
| 20 |
+
"disfluency_event": {"turn_index": 0, "subtype": "self-repair", "gap_steps": 7, "gap_seconds": 2.8, "marker_id": "repair_zh_01"},
|
| 21 |
+
"turns": [
|
| 22 |
+
{
|
| 23 |
+
"turn_index": 1,
|
| 24 |
+
"disfluency_subtype": "self-repair",
|
| 25 |
+
"source_motion_filename": "neutral_walk_ff_270_R_001__A534_M",
|
| 26 |
+
"audio_path": "/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/videogen/hfd_hesitation_pause_v1/audio/review_examples/train/repair_grab_to_left_zh_female_01_zh.wav",
|
| 27 |
+
"spoken_command": "向前……不对,我是说……向左走路",
|
| 28 |
+
"provisional_instruction": "向前走并抓取某物",
|
| 29 |
+
"final_instruction": "向左走路",
|
| 30 |
+
"ideal_motion": "整个改口过程只 listen;不得向前抓取,最终只相对当前朝向向左走。"
|
| 31 |
+
},
|
| 32 |
+
{
|
| 33 |
+
"turn_index": 2,
|
| 34 |
+
"source_motion_filename": "wave_two_hands_R_001__A533",
|
| 35 |
+
"audio_path": "/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/qwen3-tts/seed_audio/000155/female_01_zh.wav",
|
| 36 |
+
"spoken_command": "双手在身前挥动",
|
| 37 |
+
"ideal_motion": "第二轮正常挥手,验证最终意图替换没有破坏后续交互。"
|
| 38 |
+
}
|
| 39 |
+
],
|
| 40 |
+
"ideal_control_timeline": ["初始 idle", "错误前缀、改口词和完整新指令全程 listen", "只在完整最终指令后进入 motion", "不为错误前缀启动 motion"],
|
| 41 |
+
"continuity_requirements": ["旧意图与最终意图共用同一 Qwen KV,不靠清缓存作弊", "同一音色和背景", "动作与 RTC 只从最终意图开始"]
|
| 42 |
+
}
|
scripts/infer/multiturn_auto_scenario_hesitation_repair_train_v2.json
ADDED
|
@@ -0,0 +1,63 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"format": "hfd_multiturn_auto_infer_scenario_v1",
|
| 3 |
+
"name": "repair_grab_to_left_then_wave_zh",
|
| 4 |
+
"interaction_mode": "hesitation",
|
| 5 |
+
"interaction_subtype": "self-repair",
|
| 6 |
+
"data_split": "train",
|
| 7 |
+
"split_seed": 42,
|
| 8 |
+
"val_ratio": 0.1,
|
| 9 |
+
"split_source_namespace": "seed",
|
| 10 |
+
"description": "先说出抓取指令的不完整前缀,再用同音色明确改口为完整向左走;只能执行最终指令。",
|
| 11 |
+
"required_semantics_version": 24,
|
| 12 |
+
"initial_idle_steps": 5,
|
| 13 |
+
"initial_idle_seconds": 2.0,
|
| 14 |
+
"motion_frames_per_turn": 450,
|
| 15 |
+
"motion_seconds_per_turn": 9.0,
|
| 16 |
+
"inter_turn_background_steps_by_turn": [
|
| 17 |
+
25
|
| 18 |
+
],
|
| 19 |
+
"inter_turn_background_seconds_by_turn": [
|
| 20 |
+
10.0
|
| 21 |
+
],
|
| 22 |
+
"terminal_background_steps": 5,
|
| 23 |
+
"terminal_background_seconds": 2.0,
|
| 24 |
+
"disfluency_event": {
|
| 25 |
+
"turn_index": 0,
|
| 26 |
+
"subtype": "self-repair",
|
| 27 |
+
"gap_steps": 7,
|
| 28 |
+
"gap_seconds": 2.8,
|
| 29 |
+
"marker_id": "repair_zh_01",
|
| 30 |
+
"repair_style": "explicit-cue"
|
| 31 |
+
},
|
| 32 |
+
"turns": [
|
| 33 |
+
{
|
| 34 |
+
"turn_index": 1,
|
| 35 |
+
"disfluency_subtype": "self-repair",
|
| 36 |
+
"source_motion_filename": "neutral_walk_ff_270_R_001__A534_M",
|
| 37 |
+
"audio_path": "/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/videogen/hfd_hesitation_pause_v2/audio/review_examples/train/repair_grab_to_left_zh_female_01_zh.wav",
|
| 38 |
+
"spoken_command": "向前……不对,我是说……向左走路",
|
| 39 |
+
"provisional_instruction": "向前走并抓取某物",
|
| 40 |
+
"final_instruction": "向左走路",
|
| 41 |
+
"ideal_motion": "整个改口过程只 listen;不得向前抓取,最终只相对当前朝向向左走。",
|
| 42 |
+
"repair_style": "explicit-cue"
|
| 43 |
+
},
|
| 44 |
+
{
|
| 45 |
+
"turn_index": 2,
|
| 46 |
+
"source_motion_filename": "wave_two_hands_R_001__A533",
|
| 47 |
+
"audio_path": "/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/qwen3-tts/seed_audio/000155/female_01_zh.wav",
|
| 48 |
+
"spoken_command": "双手在身前挥动",
|
| 49 |
+
"ideal_motion": "第二轮正常挥手,验证最终意图替换没有破坏后续交互。"
|
| 50 |
+
}
|
| 51 |
+
],
|
| 52 |
+
"ideal_control_timeline": [
|
| 53 |
+
"初始 idle",
|
| 54 |
+
"错误前缀、改口词和完整新指令全程 listen",
|
| 55 |
+
"只在完整最终指令后进入 motion",
|
| 56 |
+
"不为错误前缀启动 motion"
|
| 57 |
+
],
|
| 58 |
+
"continuity_requirements": [
|
| 59 |
+
"旧意图与最终意图共用同一 Qwen KV,不靠清缓存作弊",
|
| 60 |
+
"同一音色和背景",
|
| 61 |
+
"动作与 RTC 只从最终意图开始"
|
| 62 |
+
]
|
| 63 |
+
}
|
scripts/infer/multiturn_auto_scenario_hesitation_repair_val_v1.json
ADDED
|
@@ -0,0 +1,42 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"format": "hfd_multiturn_auto_infer_scenario_v1",
|
| 3 |
+
"name": "repair_pick_to_jog_then_pick_en",
|
| 4 |
+
"interaction_mode": "hesitation",
|
| 5 |
+
"interaction_subtype": "self-repair",
|
| 6 |
+
"data_split": "val",
|
| 7 |
+
"split_seed": 42,
|
| 8 |
+
"val_ratio": 0.1,
|
| 9 |
+
"split_source_namespace": "seed",
|
| 10 |
+
"description": "未见 motion 上先说搬运长指令的错误前缀,再用验证集 cue 改口为完整向前慢跑。",
|
| 11 |
+
"required_semantics_version": 23,
|
| 12 |
+
"initial_idle_steps": 4,
|
| 13 |
+
"initial_idle_seconds": 1.6,
|
| 14 |
+
"motion_frames_per_turn": 450,
|
| 15 |
+
"motion_seconds_per_turn": 9.0,
|
| 16 |
+
"inter_turn_background_steps_by_turn": [25],
|
| 17 |
+
"inter_turn_background_seconds_by_turn": [10.0],
|
| 18 |
+
"terminal_background_steps": 6,
|
| 19 |
+
"terminal_background_seconds": 2.4,
|
| 20 |
+
"disfluency_event": {"turn_index": 0, "subtype": "self-repair", "gap_steps": 7, "gap_seconds": 2.8, "marker_id": "repair_en_11"},
|
| 21 |
+
"turns": [
|
| 22 |
+
{
|
| 23 |
+
"turn_index": 1,
|
| 24 |
+
"disfluency_subtype": "self-repair",
|
| 25 |
+
"source_motion_filename": "jog_ff_start_180_R_003__A068",
|
| 26 |
+
"audio_path": "/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/videogen/hfd_hesitation_pause_v1/audio/review_examples/val/repair_pick_to_jog_en_female_02_en.wav",
|
| 27 |
+
"spoken_command": "picks up ... Wait, I mean ... start jog forward",
|
| 28 |
+
"provisional_instruction": "picks up medium big light object with two hands from low right side and puts it away to the right side high",
|
| 29 |
+
"final_instruction": "start jog forward",
|
| 30 |
+
"ideal_motion": "不得执行搬运;完整听完 start jog forward 后才向前慢跑。"
|
| 31 |
+
},
|
| 32 |
+
{
|
| 33 |
+
"turn_index": 2,
|
| 34 |
+
"source_motion_filename": "medium_big_light_two_hands_right_side_low_to_right_side_high_R_001__A529",
|
| 35 |
+
"audio_path": "/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/qwen3-tts/seed_audio/000227/female_02_en.wav",
|
| 36 |
+
"spoken_command": "picks up medium big light object with two hands from low right side and puts it away to the right side high",
|
| 37 |
+
"ideal_motion": "只有第二轮真正完整重发搬运指令时,才执行右侧双手取放。"
|
| 38 |
+
}
|
| 39 |
+
],
|
| 40 |
+
"ideal_control_timeline": ["初始 idle", "假开头、Wait I mean 和完整最终指令全程 listen", "第一轮只执行 jog", "第二轮完整指令后才执行 pick"],
|
| 41 |
+
"continuity_requirements": ["验证 cue 与训练 cue 词面分离", "Qwen KV 保留旧指令但最新完整意图应覆盖它", "两条动作与同一声音身份均保持 val 审计"]
|
| 42 |
+
}
|
scripts/infer/multiturn_auto_scenario_hesitation_repair_val_v2.json
ADDED
|
@@ -0,0 +1,62 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"format": "hfd_multiturn_auto_infer_scenario_v1",
|
| 3 |
+
"name": "repair_pick_to_jog_then_pick_en",
|
| 4 |
+
"interaction_mode": "hesitation",
|
| 5 |
+
"interaction_subtype": "self-repair",
|
| 6 |
+
"data_split": "val",
|
| 7 |
+
"split_seed": 42,
|
| 8 |
+
"val_ratio": 0.1,
|
| 9 |
+
"split_source_namespace": "seed",
|
| 10 |
+
"description": "未见 motion 上先说搬运长指令的错误前缀,停顿后不使用 I mean 等提示词,直接重新说完整向前慢跑指令。",
|
| 11 |
+
"required_semantics_version": 24,
|
| 12 |
+
"initial_idle_steps": 4,
|
| 13 |
+
"initial_idle_seconds": 1.6,
|
| 14 |
+
"motion_frames_per_turn": 450,
|
| 15 |
+
"motion_seconds_per_turn": 9.0,
|
| 16 |
+
"inter_turn_background_steps_by_turn": [
|
| 17 |
+
25
|
| 18 |
+
],
|
| 19 |
+
"inter_turn_background_seconds_by_turn": [
|
| 20 |
+
10.0
|
| 21 |
+
],
|
| 22 |
+
"terminal_background_steps": 6,
|
| 23 |
+
"terminal_background_seconds": 2.4,
|
| 24 |
+
"disfluency_event": {
|
| 25 |
+
"turn_index": 0,
|
| 26 |
+
"subtype": "self-repair",
|
| 27 |
+
"gap_steps": 7,
|
| 28 |
+
"gap_seconds": 2.8,
|
| 29 |
+
"repair_style": "implicit-restart"
|
| 30 |
+
},
|
| 31 |
+
"turns": [
|
| 32 |
+
{
|
| 33 |
+
"turn_index": 1,
|
| 34 |
+
"disfluency_subtype": "self-repair",
|
| 35 |
+
"source_motion_filename": "jog_ff_start_180_R_003__A068",
|
| 36 |
+
"audio_path": "/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/videogen/hfd_hesitation_pause_v2/audio/review_examples/val/repair_pick_to_jog_en_female_02_en.wav",
|
| 37 |
+
"spoken_command": "picks up ... (2.8 s pause) ... start jog forward",
|
| 38 |
+
"provisional_instruction": "picks up medium big light object with two hands from low right side and puts it away to the right side high",
|
| 39 |
+
"final_instruction": "start jog forward",
|
| 40 |
+
"ideal_motion": "没有改口提示词也不得执行搬运;完整听完 start jog forward 后才向前慢跑。",
|
| 41 |
+
"repair_style": "implicit-restart"
|
| 42 |
+
},
|
| 43 |
+
{
|
| 44 |
+
"turn_index": 2,
|
| 45 |
+
"source_motion_filename": "medium_big_light_two_hands_right_side_low_to_right_side_high_R_001__A529",
|
| 46 |
+
"audio_path": "/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/qwen3-tts/seed_audio/000227/female_02_en.wav",
|
| 47 |
+
"spoken_command": "picks up medium big light object with two hands from low right side and puts it away to the right side high",
|
| 48 |
+
"ideal_motion": "只有第二轮真正完整重发搬运指令时,才执行右侧双手取放。"
|
| 49 |
+
}
|
| 50 |
+
],
|
| 51 |
+
"ideal_control_timeline": [
|
| 52 |
+
"初始 idle",
|
| 53 |
+
"错误开头、停顿和重新开始的完整最终指令全程 listen",
|
| 54 |
+
"第一轮只执行 jog",
|
| 55 |
+
"第二轮完整指令后才执行 pick"
|
| 56 |
+
],
|
| 57 |
+
"continuity_requirements": [
|
| 58 |
+
"不依赖 I mean 等关键词判断改口",
|
| 59 |
+
"Qwen KV 保留旧指令但最新完整意图应覆盖它",
|
| 60 |
+
"两条动作与同一声音身份均保持 val 审计"
|
| 61 |
+
]
|
| 62 |
+
}
|
scripts/infer/multiturn_auto_scenario_interruption_train_v1.json
ADDED
|
@@ -0,0 +1,60 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"format": "hfd_multiturn_auto_infer_scenario_v1",
|
| 3 |
+
"name": "interrupt_grab_wave_walk_left_zh",
|
| 4 |
+
"interaction_mode": "interruption",
|
| 5 |
+
"data_split": "train",
|
| 6 |
+
"split_seed": 42,
|
| 7 |
+
"val_ratio": 0.1,
|
| 8 |
+
"split_source_namespace": "seed",
|
| 9 |
+
"description": "三轮训练集指令;第二、三条语音分别在上一动作开始约 4.0/4.8 秒后到来,用于检查因果打断、连续 KV 和连续 RTC。",
|
| 10 |
+
"required_semantics_version": 20,
|
| 11 |
+
"initial_idle_steps": 5,
|
| 12 |
+
"initial_idle_seconds": 2.0,
|
| 13 |
+
"motion_frames_per_turn": 450,
|
| 14 |
+
"motion_seconds_per_turn": 9.0,
|
| 15 |
+
"inter_turn_background_steps_by_turn": [
|
| 16 |
+
10,
|
| 17 |
+
12
|
| 18 |
+
],
|
| 19 |
+
"inter_turn_background_seconds_by_turn": [
|
| 20 |
+
4.0,
|
| 21 |
+
4.8
|
| 22 |
+
],
|
| 23 |
+
"terminal_background_steps": 5,
|
| 24 |
+
"terminal_background_seconds": 2.0,
|
| 25 |
+
"turns": [
|
| 26 |
+
{
|
| 27 |
+
"turn_index": 1,
|
| 28 |
+
"source_motion_filename": "grab_walk_ff_180_001__A548",
|
| 29 |
+
"audio_path": "/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/qwen3-tts/seed_audio/000006/female_01_zh.wav",
|
| 30 |
+
"spoken_command": "向前走并抓取某物",
|
| 31 |
+
"ideal_motion": "听完后开始向前走并抓取;动作尚未结束时收到下一条指令。"
|
| 32 |
+
},
|
| 33 |
+
{
|
| 34 |
+
"turn_index": 2,
|
| 35 |
+
"source_motion_filename": "wave_two_hands_R_001__A533",
|
| 36 |
+
"audio_path": "/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/qwen3-tts/seed_audio/000155/female_01_zh.wav",
|
| 37 |
+
"spoken_command": "双手在身前挥动",
|
| 38 |
+
"ideal_motion": "新语音刚出现时短暂延续旧动作,听清后平滑切入 listen,再从当时真实姿态开始挥手。"
|
| 39 |
+
},
|
| 40 |
+
{
|
| 41 |
+
"turn_index": 3,
|
| 42 |
+
"source_motion_filename": "neutral_walk_ff_270_R_001__A534_M",
|
| 43 |
+
"audio_path": "/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/qwen3-tts/seed_audio/000120/female_01_zh.wav",
|
| 44 |
+
"spoken_command": "向左走路",
|
| 45 |
+
"ideal_motion": "挥手尚未自然结束时响应新语音,随后相对当前身体朝向向左行走并最终恢复站立。"
|
| 46 |
+
}
|
| 47 |
+
],
|
| 48 |
+
"ideal_control_timeline": [
|
| 49 |
+
"初始背景输出 </idle>,第一条语音出现后进入 </listen>",
|
| 50 |
+
"每次新语音刚开始时仍执行上一轮 </motion>,不提前读取未来语义",
|
| 51 |
+
"听到有效语音后由 </motion> 平滑切到 </listen>",
|
| 52 |
+
"语音结束后进入新的 </motion>,最后回到 </idle>"
|
| 53 |
+
],
|
| 54 |
+
"continuity_requirements": [
|
| 55 |
+
"三轮共用一个 Qwen KV cache 和一段连续办公室背景",
|
| 56 |
+
"打断不清空机器人状态或 RTC 历史",
|
| 57 |
+
"新动作从旧动作被打断时的真实位置、朝向和关节状态接续",
|
| 58 |
+
"旧动作的 future progress 仍以其原始完整时长为分母"
|
| 59 |
+
]
|
| 60 |
+
}
|
scripts/infer/multiturn_auto_scenario_pause_train_v1.json
ADDED
|
@@ -0,0 +1,41 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"format": "hfd_multiturn_auto_infer_scenario_v1",
|
| 3 |
+
"name": "pause_grab_then_wave_zh",
|
| 4 |
+
"interaction_mode": "pause",
|
| 5 |
+
"interaction_subtype": "silent-pause",
|
| 6 |
+
"data_split": "train",
|
| 7 |
+
"split_seed": 42,
|
| 8 |
+
"val_ratio": 0.1,
|
| 9 |
+
"split_source_namespace": "seed",
|
| 10 |
+
"description": "训练集中文指令在句中静默 1.6 秒;停顿期间继续 listen,完整意图结束后才抓取,随后正常接收第二轮指令。",
|
| 11 |
+
"required_semantics_version": 23,
|
| 12 |
+
"initial_idle_steps": 5,
|
| 13 |
+
"initial_idle_seconds": 2.0,
|
| 14 |
+
"motion_frames_per_turn": 450,
|
| 15 |
+
"motion_seconds_per_turn": 9.0,
|
| 16 |
+
"inter_turn_background_steps_by_turn": [25],
|
| 17 |
+
"inter_turn_background_seconds_by_turn": [10.0],
|
| 18 |
+
"terminal_background_steps": 5,
|
| 19 |
+
"terminal_background_seconds": 2.0,
|
| 20 |
+
"disfluency_event": {"turn_index": 0, "subtype": "silent-pause", "gap_steps": 4, "gap_seconds": 1.6},
|
| 21 |
+
"turns": [
|
| 22 |
+
{
|
| 23 |
+
"turn_index": 1,
|
| 24 |
+
"disfluency_subtype": "silent-pause",
|
| 25 |
+
"source_motion_filename": "grab_walk_ff_180_001__A548",
|
| 26 |
+
"audio_path": "/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/videogen/hfd_hesitation_pause_v1/audio/review_examples/train/pause_grab_zh_female_01_zh.wav",
|
| 27 |
+
"spoken_command": "向前……(静默思考 1.6 秒)……走并抓取某物",
|
| 28 |
+
"final_instruction": "向前走并抓取某物",
|
| 29 |
+
"ideal_motion": "静默段全程保持自然聆听,不提前走;听到完整后向前走并抓取。"
|
| 30 |
+
},
|
| 31 |
+
{
|
| 32 |
+
"turn_index": 2,
|
| 33 |
+
"source_motion_filename": "wave_two_hands_R_001__A533",
|
| 34 |
+
"audio_path": "/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/qwen3-tts/seed_audio/000155/female_01_zh.wav",
|
| 35 |
+
"spoken_command": "双手在身前挥动",
|
| 36 |
+
"ideal_motion": "从第一轮结束状态自然进入第二轮双手挥动。"
|
| 37 |
+
}
|
| 38 |
+
],
|
| 39 |
+
"ideal_control_timeline": ["背景为 idle", "语音开始后进入 listen", "句中静默仍保持 listen", "完整指令结束后进入 motion", "下一条完整指令按普通 turn-taking 执行"],
|
| 40 |
+
"continuity_requirements": ["整段 session 只使用一个 Qwen KV", "同一连续办公室背景", "RTC、位置和朝向不在停顿或轮次边界重置"]
|
| 41 |
+
}
|
scripts/infer/multiturn_auto_scenario_pause_train_v2.json
ADDED
|
@@ -0,0 +1,60 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"format": "hfd_multiturn_auto_infer_scenario_v1",
|
| 3 |
+
"name": "pause_grab_then_wave_zh",
|
| 4 |
+
"interaction_mode": "pause",
|
| 5 |
+
"interaction_subtype": "silent-pause",
|
| 6 |
+
"data_split": "train",
|
| 7 |
+
"split_seed": 42,
|
| 8 |
+
"val_ratio": 0.1,
|
| 9 |
+
"split_source_namespace": "seed",
|
| 10 |
+
"description": "训练集中文指令在句中静默 1.6 秒;停顿期间继续 listen,完整意图结束后才抓取,随后正常接收第二轮指令。",
|
| 11 |
+
"required_semantics_version": 24,
|
| 12 |
+
"initial_idle_steps": 5,
|
| 13 |
+
"initial_idle_seconds": 2.0,
|
| 14 |
+
"motion_frames_per_turn": 450,
|
| 15 |
+
"motion_seconds_per_turn": 9.0,
|
| 16 |
+
"inter_turn_background_steps_by_turn": [
|
| 17 |
+
25
|
| 18 |
+
],
|
| 19 |
+
"inter_turn_background_seconds_by_turn": [
|
| 20 |
+
10.0
|
| 21 |
+
],
|
| 22 |
+
"terminal_background_steps": 5,
|
| 23 |
+
"terminal_background_seconds": 2.0,
|
| 24 |
+
"disfluency_event": {
|
| 25 |
+
"turn_index": 0,
|
| 26 |
+
"subtype": "silent-pause",
|
| 27 |
+
"gap_steps": 4,
|
| 28 |
+
"gap_seconds": 1.6
|
| 29 |
+
},
|
| 30 |
+
"turns": [
|
| 31 |
+
{
|
| 32 |
+
"turn_index": 1,
|
| 33 |
+
"disfluency_subtype": "silent-pause",
|
| 34 |
+
"source_motion_filename": "grab_walk_ff_180_001__A548",
|
| 35 |
+
"audio_path": "/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/videogen/hfd_hesitation_pause_v2/audio/review_examples/train/pause_grab_zh_female_01_zh.wav",
|
| 36 |
+
"spoken_command": "向前……(静默思考 1.6 秒)……走并抓取某物",
|
| 37 |
+
"final_instruction": "向前走并抓取某物",
|
| 38 |
+
"ideal_motion": "静默段全程保持自然聆听,不提前走;听到完整后向前走并抓取。"
|
| 39 |
+
},
|
| 40 |
+
{
|
| 41 |
+
"turn_index": 2,
|
| 42 |
+
"source_motion_filename": "wave_two_hands_R_001__A533",
|
| 43 |
+
"audio_path": "/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/qwen3-tts/seed_audio/000155/female_01_zh.wav",
|
| 44 |
+
"spoken_command": "双手在身前挥动",
|
| 45 |
+
"ideal_motion": "从第一轮结束状态自然进入第二轮双手挥动。"
|
| 46 |
+
}
|
| 47 |
+
],
|
| 48 |
+
"ideal_control_timeline": [
|
| 49 |
+
"背景为 idle",
|
| 50 |
+
"语音开始后进入 listen",
|
| 51 |
+
"句中静默仍保持 listen",
|
| 52 |
+
"完整指令结束后进入 motion",
|
| 53 |
+
"下一条完整指令按普通 turn-taking 执行"
|
| 54 |
+
],
|
| 55 |
+
"continuity_requirements": [
|
| 56 |
+
"整段 session 只使用一个 Qwen KV",
|
| 57 |
+
"同一连续办公室背景",
|
| 58 |
+
"RTC、位置和朝向不在停顿或轮次边界重置"
|
| 59 |
+
]
|
| 60 |
+
}
|
scripts/infer/multiturn_auto_scenario_safe_hold_val_v1.json
ADDED
|
@@ -0,0 +1,40 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"format": "hfd_multiturn_auto_infer_scenario_v1",
|
| 3 |
+
"name": "hold_val_jump_en",
|
| 4 |
+
"interaction_mode": "safe-hold",
|
| 5 |
+
"data_split": "val",
|
| 6 |
+
"split_seed": 42,
|
| 7 |
+
"val_ratio": 0.1,
|
| 8 |
+
"split_source_namespace": "seed",
|
| 9 |
+
"description": "未见过的跳跃 motion 中收到保持不动,检查是否先落地稳定再保持。",
|
| 10 |
+
"required_semantics_version": 21,
|
| 11 |
+
"initial_idle_steps": 4,
|
| 12 |
+
"initial_idle_seconds": 1.6,
|
| 13 |
+
"motion_frames_per_turn": 450,
|
| 14 |
+
"motion_seconds_per_turn": 9.0,
|
| 15 |
+
"inter_turn_background_steps_by_turn": [7],
|
| 16 |
+
"inter_turn_background_seconds_by_turn": [2.8],
|
| 17 |
+
"terminal_background_steps": 6,
|
| 18 |
+
"terminal_background_seconds": 2.4,
|
| 19 |
+
"turns": [
|
| 20 |
+
{
|
| 21 |
+
"turn_index": 1,
|
| 22 |
+
"source_motion_filename": "reach_jump_R_003__A148_M",
|
| 23 |
+
"audio_path": "/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/qwen3-tts/seed_audio/000243/female_02_en.wav",
|
| 24 |
+
"spoken_command": "reaching something during jump",
|
| 25 |
+
"ideal_motion": "起跳并在空中伸手。"
|
| 26 |
+
},
|
| 27 |
+
{
|
| 28 |
+
"turn_index": 2,
|
| 29 |
+
"control_intent": "safe-hold",
|
| 30 |
+
"audio_path": "/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/videogen/hfd_control_commands_v1/audio/safe-hold/hold_06/female_02_en.wav",
|
| 31 |
+
"spoken_command": "Stay balanced and don't move.",
|
| 32 |
+
"ideal_motion": "如果正在腾空或落地中则先完成安全落地,然后保持真实稳定姿态,不回零到预设 idle。"
|
| 33 |
+
}
|
| 34 |
+
],
|
| 35 |
+
"ideal_control_timeline": [
|
| 36 |
+
"idle -> listen -> motion",
|
| 37 |
+
"hold 语音在 motion 中到来,没有 listen",
|
| 38 |
+
"不稳定时保持 motion,稳定后转 idle 并保持当时姿"
|
| 39 |
+
]
|
| 40 |
+
}
|
scripts/infer/multiturn_auto_scenario_v1.json
ADDED
|
@@ -0,0 +1,61 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"format": "hfd_multiturn_auto_infer_scenario_v1",
|
| 3 |
+
"name": "grab_wave_walk_left_zh",
|
| 4 |
+
"interaction_mode": "turn-taking",
|
| 5 |
+
"data_split": "train",
|
| 6 |
+
"split_seed": 42,
|
| 7 |
+
"val_ratio": 0.1,
|
| 8 |
+
"split_source_namespace": "seed",
|
| 9 |
+
"description": "三轮连续中文指令;同一机器人状态、办公室背景、RTC 历史和 Qwen KV 从头保留到尾。",
|
| 10 |
+
"required_semantics_version": 20,
|
| 11 |
+
"initial_idle_steps": 5,
|
| 12 |
+
"initial_idle_seconds": 2.0,
|
| 13 |
+
"motion_frames_per_turn": 450,
|
| 14 |
+
"motion_seconds_per_turn": 9.0,
|
| 15 |
+
"inter_turn_background_steps_by_turn": [
|
| 16 |
+
25,
|
| 17 |
+
30
|
| 18 |
+
],
|
| 19 |
+
"inter_turn_background_seconds_by_turn": [
|
| 20 |
+
10.0,
|
| 21 |
+
12.0
|
| 22 |
+
],
|
| 23 |
+
"terminal_background_steps": 5,
|
| 24 |
+
"terminal_background_seconds": 2.0,
|
| 25 |
+
"turns": [
|
| 26 |
+
{
|
| 27 |
+
"turn_index": 1,
|
| 28 |
+
"source_motion_filename": "grab_walk_ff_180_001__A548",
|
| 29 |
+
"audio_path": "/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/qwen3-tts/seed_audio/000006/female_01_zh.wav",
|
| 30 |
+
"spoken_command": "向前走并抓取某物",
|
| 31 |
+
"ideal_motion": "听完后向当前身体前方行走,同时完成清晰的伸手抓取;动作结束后在新位置恢复正常站立。"
|
| 32 |
+
},
|
| 33 |
+
{
|
| 34 |
+
"turn_index": 2,
|
| 35 |
+
"source_motion_filename": "wave_two_hands_R_001__A533",
|
| 36 |
+
"audio_path": "/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/qwen3-tts/seed_audio/000155/female_01_zh.wav",
|
| 37 |
+
"spoken_command": "双手在身前挥动",
|
| 38 |
+
"ideal_motion": "保持第一轮累计的位置和朝向,身体基本原地站稳,双手在身前挥动,然后自然收手恢复站立。"
|
| 39 |
+
},
|
| 40 |
+
{
|
| 41 |
+
"turn_index": 3,
|
| 42 |
+
"source_motion_filename": "neutral_walk_ff_270_R_001__A534_M",
|
| 43 |
+
"audio_path": "/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/qwen3-tts/seed_audio/000120/female_01_zh.wav",
|
| 44 |
+
"spoken_command": "向左走路",
|
| 45 |
+
"ideal_motion": "相对第二轮结束时的身体朝向向左侧行走,不重置到世界原点,最后恢复正常站立。"
|
| 46 |
+
}
|
| 47 |
+
],
|
| 48 |
+
"ideal_control_timeline": [
|
| 49 |
+
"初始纯背景时输出 </idle>,机器人自然站立且不提前抬手或移动",
|
| 50 |
+
"每次有人说话时切到 </listen>,保持自然聆听动作",
|
| 51 |
+
"每条指令结束后切到 </motion>,完整执行该轮动作",
|
| 52 |
+
"第二轮前允许短暂停顿,第三轮前包含约 2 秒额外 idle",
|
| 53 |
+
"最后一轮结束后,在 2 秒纯背景中回到并持续输出 </idle>"
|
| 54 |
+
],
|
| 55 |
+
"continuity_requirements": [
|
| 56 |
+
"三轮只使用一个 Qwen KV cache,不在轮次边界清空",
|
| 57 |
+
"三轮使用同一段连续办公室背景,不拼接不同背景音",
|
| 58 |
+
"后续动作从上一轮真实生成的最终位置、朝向和关节状态继续",
|
| 59 |
+
"RTC 历史按真实执行的动作连续传递,不读取 GT,也不重置机器人"
|
| 60 |
+
]
|
| 61 |
+
}
|
scripts/infer/multiturn_auto_scenario_val_v1.json
ADDED
|
@@ -0,0 +1,100 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"format": "hfd_multiturn_auto_infer_scenario_v1",
|
| 3 |
+
"name": "val_jog_pick_jump_idle_en_timeline_v2",
|
| 4 |
+
"interaction_mode": "turn-taking",
|
| 5 |
+
"description": "训练时 hash 划分到 val 的第 1199 条四轮 session;四条具体 motion source 均未进入训练集,并严格复现该 session 的真实语音起点。",
|
| 6 |
+
"data_split": "val",
|
| 7 |
+
"split_seed": 42,
|
| 8 |
+
"val_ratio": 0.1,
|
| 9 |
+
"split_source_namespace": "seed",
|
| 10 |
+
"validation_session_index": 1199,
|
| 11 |
+
"required_semantics_version": 20,
|
| 12 |
+
"initial_idle_steps": 4,
|
| 13 |
+
"initial_idle_seconds": 1.6,
|
| 14 |
+
"motion_frames_per_turn": 450,
|
| 15 |
+
"motion_seconds_per_turn": 9.0,
|
| 16 |
+
"inter_turn_background_steps_by_turn": [
|
| 17 |
+
18,
|
| 18 |
+
32,
|
| 19 |
+
14
|
| 20 |
+
],
|
| 21 |
+
"inter_turn_background_seconds_by_turn": [
|
| 22 |
+
7.2,
|
| 23 |
+
12.8,
|
| 24 |
+
5.6
|
| 25 |
+
],
|
| 26 |
+
"turn_audio_window_counts": [
|
| 27 |
+
4,
|
| 28 |
+
18,
|
| 29 |
+
5,
|
| 30 |
+
10
|
| 31 |
+
],
|
| 32 |
+
"expected_turn_window_starts": [
|
| 33 |
+
0,
|
| 34 |
+
22,
|
| 35 |
+
72,
|
| 36 |
+
91
|
| 37 |
+
],
|
| 38 |
+
"source_turn_speech_start_frames_50fps": [
|
| 39 |
+
80,
|
| 40 |
+
520,
|
| 41 |
+
1520,
|
| 42 |
+
1900
|
| 43 |
+
],
|
| 44 |
+
"source_turn_listen_start_frames_50fps": [
|
| 45 |
+
120,
|
| 46 |
+
560,
|
| 47 |
+
1560,
|
| 48 |
+
1940
|
| 49 |
+
],
|
| 50 |
+
"source_inter_turn_idle_steps": [
|
| 51 |
+
0,
|
| 52 |
+
8,
|
| 53 |
+
0
|
| 54 |
+
],
|
| 55 |
+
"terminal_background_steps": 6,
|
| 56 |
+
"terminal_background_seconds": 2.4,
|
| 57 |
+
"turns": [
|
| 58 |
+
{
|
| 59 |
+
"turn_index": 1,
|
| 60 |
+
"source_motion_filename": "jog_ff_start_180_R_003__A068",
|
| 61 |
+
"audio_path": "/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/qwen3-tts/seed_audio/000175/female_02_en.wav",
|
| 62 |
+
"spoken_command": "start jog forward",
|
| 63 |
+
"ideal_motion": "听完英文指令后从当前站立状态开始向身体前方慢跑,保持朝向连续。"
|
| 64 |
+
},
|
| 65 |
+
{
|
| 66 |
+
"turn_index": 2,
|
| 67 |
+
"source_motion_filename": "medium_big_light_two_hands_right_side_low_to_right_side_high_R_001__A529",
|
| 68 |
+
"audio_path": "/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/qwen3-tts/seed_audio/000227/female_02_en.wav",
|
| 69 |
+
"spoken_command": "picks up medium big light object with two hands from low right side and puts it away to the right side high",
|
| 70 |
+
"ideal_motion": "在第一轮最终位置和朝向上,用双手从右下方拿起物体并举到右上方;这条完整指令和音频目录训练时从未出现。"
|
| 71 |
+
},
|
| 72 |
+
{
|
| 73 |
+
"turn_index": 3,
|
| 74 |
+
"source_motion_filename": "reach_jump_R_003__A148_M",
|
| 75 |
+
"audio_path": "/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/qwen3-tts/seed_audio/000243/female_02_en.wav",
|
| 76 |
+
"spoken_command": "reaching something during jump",
|
| 77 |
+
"ideal_motion": "保持累计状态,完成带伸手够物意图的跳跃并稳定落地。"
|
| 78 |
+
},
|
| 79 |
+
{
|
| 80 |
+
"turn_index": 4,
|
| 81 |
+
"source_motion_filename": "idle_right_to_idle_003__A094_M",
|
| 82 |
+
"audio_path": "/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/qwen3-tts/seed_audio/003713/female_02_en.wav",
|
| 83 |
+
"spoken_command": "from neutral position with left leg forward to neutral position",
|
| 84 |
+
"ideal_motion": "从左腿略在前的自然站姿平稳回到中立站姿,随后在纯背景中回到 idle。"
|
| 85 |
+
}
|
| 86 |
+
],
|
| 87 |
+
"ideal_control_timeline": [
|
| 88 |
+
"初始纯背景输出 </idle>",
|
| 89 |
+
"每轮语音出现后切换到 </listen>",
|
| 90 |
+
"听清完整指令后切换到 </motion>",
|
| 91 |
+
"四轮语音起点严格复现验证 source 的 0、8.8、28.8、36.4 秒相对时间;7.2、12.8、5.6 秒是扣除上一条语音窗口后的实际背景间隔",
|
| 92 |
+
"第四轮结束后在 2.4 秒背景中回到 </idle>"
|
| 93 |
+
],
|
| 94 |
+
"continuity_requirements": [
|
| 95 |
+
"四轮共用一个 Qwen KV cache",
|
| 96 |
+
"四轮使用同一段连续办公室背景",
|
| 97 |
+
"机器人位置、朝向、关节状态和 RTC 历史不在轮次间重置",
|
| 98 |
+
"所有 source_motion_filename 均按训练 run 的 split_seed=42、val_ratio=0.1 落在 val"
|
| 99 |
+
]
|
| 100 |
+
}
|
scripts/infer/multiturn_auto_suite_v2.json
ADDED
|
@@ -0,0 +1,92 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"format": "hfd_multiturn_auto_infer_suite_v2",
|
| 3 |
+
"description": "同一 checkpoint 的既有五类任务及 pause、filled hesitation、self-repair hesitation × train/val 共十六组闭环推理;由空闲 GPU 动态领取任务。",
|
| 4 |
+
"cases": [
|
| 5 |
+
{
|
| 6 |
+
"interaction_mode": "turn-taking",
|
| 7 |
+
"data_split": "train",
|
| 8 |
+
"scenario_file": "multiturn_auto_scenario_v1.json"
|
| 9 |
+
},
|
| 10 |
+
{
|
| 11 |
+
"interaction_mode": "turn-taking",
|
| 12 |
+
"data_split": "val",
|
| 13 |
+
"scenario_file": "multiturn_auto_scenario_val_v1.json"
|
| 14 |
+
},
|
| 15 |
+
{
|
| 16 |
+
"interaction_mode": "interruption",
|
| 17 |
+
"data_split": "train",
|
| 18 |
+
"scenario_file": "multiturn_auto_scenario_interruption_train_v1.json"
|
| 19 |
+
},
|
| 20 |
+
{
|
| 21 |
+
"interaction_mode": "interruption",
|
| 22 |
+
"data_split": "val",
|
| 23 |
+
"scenario_file": "multiturn_auto_scenario_interruption_val_v1.json"
|
| 24 |
+
},
|
| 25 |
+
{
|
| 26 |
+
"interaction_mode": "emergency-stop",
|
| 27 |
+
"data_split": "train",
|
| 28 |
+
"scenario_file": "multiturn_auto_scenario_emergency_stop_train_v1.json"
|
| 29 |
+
},
|
| 30 |
+
{
|
| 31 |
+
"interaction_mode": "emergency-stop",
|
| 32 |
+
"data_split": "val",
|
| 33 |
+
"scenario_file": "multiturn_auto_scenario_emergency_stop_val_v1.json"
|
| 34 |
+
},
|
| 35 |
+
{
|
| 36 |
+
"interaction_mode": "safe-hold",
|
| 37 |
+
"data_split": "train",
|
| 38 |
+
"scenario_file": "multiturn_auto_scenario_safe_hold_train_v1.json"
|
| 39 |
+
},
|
| 40 |
+
{
|
| 41 |
+
"interaction_mode": "safe-hold",
|
| 42 |
+
"data_split": "val",
|
| 43 |
+
"scenario_file": "multiturn_auto_scenario_safe_hold_val_v1.json"
|
| 44 |
+
},
|
| 45 |
+
{
|
| 46 |
+
"interaction_mode": "backchannel",
|
| 47 |
+
"data_split": "train",
|
| 48 |
+
"scenario_file": "multiturn_auto_scenario_backchannel_train_v1.json"
|
| 49 |
+
},
|
| 50 |
+
{
|
| 51 |
+
"interaction_mode": "backchannel",
|
| 52 |
+
"data_split": "val",
|
| 53 |
+
"scenario_file": "multiturn_auto_scenario_backchannel_val_v1.json"
|
| 54 |
+
},
|
| 55 |
+
{
|
| 56 |
+
"interaction_mode": "pause",
|
| 57 |
+
"interaction_subtype": "silent-pause",
|
| 58 |
+
"data_split": "train",
|
| 59 |
+
"scenario_file": "multiturn_auto_scenario_pause_train_v1.json"
|
| 60 |
+
},
|
| 61 |
+
{
|
| 62 |
+
"interaction_mode": "pause",
|
| 63 |
+
"interaction_subtype": "silent-pause",
|
| 64 |
+
"data_split": "val",
|
| 65 |
+
"scenario_file": "multiturn_auto_scenario_pause_val_v1.json"
|
| 66 |
+
},
|
| 67 |
+
{
|
| 68 |
+
"interaction_mode": "hesitation",
|
| 69 |
+
"interaction_subtype": "filled-pause",
|
| 70 |
+
"data_split": "train",
|
| 71 |
+
"scenario_file": "multiturn_auto_scenario_hesitation_filled_train_v1.json"
|
| 72 |
+
},
|
| 73 |
+
{
|
| 74 |
+
"interaction_mode": "hesitation",
|
| 75 |
+
"interaction_subtype": "filled-pause",
|
| 76 |
+
"data_split": "val",
|
| 77 |
+
"scenario_file": "multiturn_auto_scenario_hesitation_filled_val_v1.json"
|
| 78 |
+
},
|
| 79 |
+
{
|
| 80 |
+
"interaction_mode": "hesitation",
|
| 81 |
+
"interaction_subtype": "self-repair",
|
| 82 |
+
"data_split": "train",
|
| 83 |
+
"scenario_file": "multiturn_auto_scenario_hesitation_repair_train_v1.json"
|
| 84 |
+
},
|
| 85 |
+
{
|
| 86 |
+
"interaction_mode": "hesitation",
|
| 87 |
+
"interaction_subtype": "self-repair",
|
| 88 |
+
"data_split": "val",
|
| 89 |
+
"scenario_file": "multiturn_auto_scenario_hesitation_repair_val_v1.json"
|
| 90 |
+
}
|
| 91 |
+
]
|
| 92 |
+
}
|
scripts/infer/multiturn_auto_suite_v3.json
ADDED
|
@@ -0,0 +1,92 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"format": "hfd_multiturn_auto_infer_suite_v3",
|
| 3 |
+
"description": "同一 checkpoint 的七类任务 × train/val 共十六组闭环推理;v24 使用自然拖长 filler,并同时覆盖显式提示和无提示词自我改口;由空闲 GPU 动态领取任务。",
|
| 4 |
+
"cases": [
|
| 5 |
+
{
|
| 6 |
+
"interaction_mode": "turn-taking",
|
| 7 |
+
"data_split": "train",
|
| 8 |
+
"scenario_file": "multiturn_auto_scenario_v1.json"
|
| 9 |
+
},
|
| 10 |
+
{
|
| 11 |
+
"interaction_mode": "turn-taking",
|
| 12 |
+
"data_split": "val",
|
| 13 |
+
"scenario_file": "multiturn_auto_scenario_val_v1.json"
|
| 14 |
+
},
|
| 15 |
+
{
|
| 16 |
+
"interaction_mode": "interruption",
|
| 17 |
+
"data_split": "train",
|
| 18 |
+
"scenario_file": "multiturn_auto_scenario_interruption_train_v1.json"
|
| 19 |
+
},
|
| 20 |
+
{
|
| 21 |
+
"interaction_mode": "interruption",
|
| 22 |
+
"data_split": "val",
|
| 23 |
+
"scenario_file": "multiturn_auto_scenario_interruption_val_v1.json"
|
| 24 |
+
},
|
| 25 |
+
{
|
| 26 |
+
"interaction_mode": "emergency-stop",
|
| 27 |
+
"data_split": "train",
|
| 28 |
+
"scenario_file": "multiturn_auto_scenario_emergency_stop_train_v1.json"
|
| 29 |
+
},
|
| 30 |
+
{
|
| 31 |
+
"interaction_mode": "emergency-stop",
|
| 32 |
+
"data_split": "val",
|
| 33 |
+
"scenario_file": "multiturn_auto_scenario_emergency_stop_val_v1.json"
|
| 34 |
+
},
|
| 35 |
+
{
|
| 36 |
+
"interaction_mode": "safe-hold",
|
| 37 |
+
"data_split": "train",
|
| 38 |
+
"scenario_file": "multiturn_auto_scenario_safe_hold_train_v1.json"
|
| 39 |
+
},
|
| 40 |
+
{
|
| 41 |
+
"interaction_mode": "safe-hold",
|
| 42 |
+
"data_split": "val",
|
| 43 |
+
"scenario_file": "multiturn_auto_scenario_safe_hold_val_v1.json"
|
| 44 |
+
},
|
| 45 |
+
{
|
| 46 |
+
"interaction_mode": "backchannel",
|
| 47 |
+
"data_split": "train",
|
| 48 |
+
"scenario_file": "multiturn_auto_scenario_backchannel_train_v1.json"
|
| 49 |
+
},
|
| 50 |
+
{
|
| 51 |
+
"interaction_mode": "backchannel",
|
| 52 |
+
"data_split": "val",
|
| 53 |
+
"scenario_file": "multiturn_auto_scenario_backchannel_val_v1.json"
|
| 54 |
+
},
|
| 55 |
+
{
|
| 56 |
+
"interaction_mode": "pause",
|
| 57 |
+
"interaction_subtype": "silent-pause",
|
| 58 |
+
"data_split": "train",
|
| 59 |
+
"scenario_file": "multiturn_auto_scenario_pause_train_v2.json"
|
| 60 |
+
},
|
| 61 |
+
{
|
| 62 |
+
"interaction_mode": "pause",
|
| 63 |
+
"interaction_subtype": "silent-pause",
|
| 64 |
+
"data_split": "val",
|
| 65 |
+
"scenario_file": "multiturn_auto_scenario_pause_val_v2.json"
|
| 66 |
+
},
|
| 67 |
+
{
|
| 68 |
+
"interaction_mode": "hesitation",
|
| 69 |
+
"interaction_subtype": "filled-pause",
|
| 70 |
+
"data_split": "train",
|
| 71 |
+
"scenario_file": "multiturn_auto_scenario_hesitation_filled_train_v2.json"
|
| 72 |
+
},
|
| 73 |
+
{
|
| 74 |
+
"interaction_mode": "hesitation",
|
| 75 |
+
"interaction_subtype": "filled-pause",
|
| 76 |
+
"data_split": "val",
|
| 77 |
+
"scenario_file": "multiturn_auto_scenario_hesitation_filled_val_v2.json"
|
| 78 |
+
},
|
| 79 |
+
{
|
| 80 |
+
"interaction_mode": "hesitation",
|
| 81 |
+
"interaction_subtype": "self-repair",
|
| 82 |
+
"data_split": "train",
|
| 83 |
+
"scenario_file": "multiturn_auto_scenario_hesitation_repair_train_v2.json"
|
| 84 |
+
},
|
| 85 |
+
{
|
| 86 |
+
"interaction_mode": "hesitation",
|
| 87 |
+
"interaction_subtype": "self-repair",
|
| 88 |
+
"data_split": "val",
|
| 89 |
+
"scenario_file": "multiturn_auto_scenario_hesitation_repair_val_v2.json"
|
| 90 |
+
}
|
| 91 |
+
]
|
| 92 |
+
}
|
scripts/infer/mux_full_duplex_audio.py
ADDED
|
@@ -0,0 +1,180 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#!/usr/bin/env python3
|
| 2 |
+
from __future__ import annotations
|
| 3 |
+
|
| 4 |
+
import argparse
|
| 5 |
+
import json
|
| 6 |
+
import subprocess
|
| 7 |
+
from pathlib import Path
|
| 8 |
+
from typing import Any
|
| 9 |
+
|
| 10 |
+
|
| 11 |
+
def probe(path: Path) -> dict[str, Any]:
|
| 12 |
+
result = subprocess.run(
|
| 13 |
+
[
|
| 14 |
+
"ffprobe",
|
| 15 |
+
"-v",
|
| 16 |
+
"error",
|
| 17 |
+
"-show_streams",
|
| 18 |
+
"-show_format",
|
| 19 |
+
"-of",
|
| 20 |
+
"json",
|
| 21 |
+
str(path),
|
| 22 |
+
],
|
| 23 |
+
capture_output=True,
|
| 24 |
+
check=False,
|
| 25 |
+
)
|
| 26 |
+
if result.returncode != 0:
|
| 27 |
+
raise RuntimeError(result.stderr.decode("utf-8", errors="ignore"))
|
| 28 |
+
return json.loads(result.stdout)
|
| 29 |
+
|
| 30 |
+
|
| 31 |
+
def stream(payload: dict[str, Any], codec_type: str) -> dict[str, Any]:
|
| 32 |
+
for item in payload.get("streams", []):
|
| 33 |
+
if item.get("codec_type") == codec_type:
|
| 34 |
+
return item
|
| 35 |
+
raise RuntimeError(f"ffprobe found no {codec_type} stream")
|
| 36 |
+
|
| 37 |
+
|
| 38 |
+
def fraction(value: str) -> float:
|
| 39 |
+
numerator, denominator = str(value).split("/", 1)
|
| 40 |
+
return float(numerator) / float(denominator)
|
| 41 |
+
|
| 42 |
+
|
| 43 |
+
def main() -> None:
|
| 44 |
+
parser = argparse.ArgumentParser(
|
| 45 |
+
description=(
|
| 46 |
+
"Mux either a complete HFD streaming audio timeline or legacy "
|
| 47 |
+
"speech-only audio onto the rendered motion"
|
| 48 |
+
)
|
| 49 |
+
)
|
| 50 |
+
parser.add_argument("--video", required=True)
|
| 51 |
+
parser.add_argument("--audio", required=True)
|
| 52 |
+
parser.add_argument("--output", required=True)
|
| 53 |
+
parser.add_argument("--infer-meta", required=True)
|
| 54 |
+
parser.add_argument(
|
| 55 |
+
"--audio-is-timeline",
|
| 56 |
+
action="store_true",
|
| 57 |
+
help=(
|
| 58 |
+
"The audio already contains initial background, speech, and "
|
| 59 |
+
"post-WAV background on the complete 50 Hz output timeline."
|
| 60 |
+
),
|
| 61 |
+
)
|
| 62 |
+
parser.add_argument("--expected-fps", type=float, default=50.0)
|
| 63 |
+
args = parser.parse_args()
|
| 64 |
+
|
| 65 |
+
video = Path(args.video).expanduser().resolve()
|
| 66 |
+
audio = Path(args.audio).expanduser().resolve()
|
| 67 |
+
output = Path(args.output).expanduser().resolve()
|
| 68 |
+
meta_path = Path(args.infer_meta).expanduser().resolve()
|
| 69 |
+
for path in (video, audio, meta_path):
|
| 70 |
+
if not path.is_file():
|
| 71 |
+
raise FileNotFoundError(path)
|
| 72 |
+
|
| 73 |
+
video_probe = probe(video)
|
| 74 |
+
video_stream = stream(video_probe, "video")
|
| 75 |
+
video_duration = float(
|
| 76 |
+
video_stream.get("duration") or video_probe["format"].get("duration")
|
| 77 |
+
)
|
| 78 |
+
video_fps = fraction(video_stream.get("avg_frame_rate") or video_stream["r_frame_rate"])
|
| 79 |
+
if abs(video_fps - float(args.expected_fps)) > 0.01:
|
| 80 |
+
raise RuntimeError(
|
| 81 |
+
f"Rendered video FPS {video_fps} does not preserve the 50 Hz HFD timeline"
|
| 82 |
+
)
|
| 83 |
+
|
| 84 |
+
metadata = json.loads(meta_path.read_text(encoding="utf-8"))
|
| 85 |
+
requested_speech_delay_seconds = max(
|
| 86 |
+
float(metadata.get("speech_start_seconds", 0.0)),
|
| 87 |
+
0.0,
|
| 88 |
+
)
|
| 89 |
+
applied_audio_delay_seconds = (
|
| 90 |
+
0.0 if bool(args.audio_is_timeline) else requested_speech_delay_seconds
|
| 91 |
+
)
|
| 92 |
+
speech_delay_ms = int(round(applied_audio_delay_seconds * 1000.0))
|
| 93 |
+
delay_filter = (
|
| 94 |
+
f"adelay={speech_delay_ms}:all=1,"
|
| 95 |
+
if speech_delay_ms > 0
|
| 96 |
+
else ""
|
| 97 |
+
)
|
| 98 |
+
output.parent.mkdir(parents=True, exist_ok=True)
|
| 99 |
+
temporary = output.with_name(f".{output.name}.tmp.mp4")
|
| 100 |
+
command = [
|
| 101 |
+
"ffmpeg",
|
| 102 |
+
"-v",
|
| 103 |
+
"error",
|
| 104 |
+
"-y",
|
| 105 |
+
"-i",
|
| 106 |
+
str(video),
|
| 107 |
+
"-i",
|
| 108 |
+
str(audio),
|
| 109 |
+
"-filter_complex",
|
| 110 |
+
f"[1:a]{delay_filter}aresample=async=1:first_pts=0,"
|
| 111 |
+
f"apad=whole_dur={video_duration:.9f}[audio]",
|
| 112 |
+
"-map",
|
| 113 |
+
"0:v:0",
|
| 114 |
+
"-map",
|
| 115 |
+
"[audio]",
|
| 116 |
+
"-c:v",
|
| 117 |
+
"copy",
|
| 118 |
+
"-c:a",
|
| 119 |
+
"aac",
|
| 120 |
+
"-movflags",
|
| 121 |
+
"+faststart",
|
| 122 |
+
"-t",
|
| 123 |
+
f"{video_duration:.9f}",
|
| 124 |
+
str(temporary),
|
| 125 |
+
]
|
| 126 |
+
result = subprocess.run(command, capture_output=True, check=False)
|
| 127 |
+
if result.returncode != 0:
|
| 128 |
+
raise RuntimeError(
|
| 129 |
+
f"ffmpeg mux failed: {result.stderr.decode('utf-8', errors='ignore')}"
|
| 130 |
+
)
|
| 131 |
+
temporary.replace(output)
|
| 132 |
+
|
| 133 |
+
output_probe = probe(output)
|
| 134 |
+
output_video = stream(output_probe, "video")
|
| 135 |
+
output_audio = stream(output_probe, "audio")
|
| 136 |
+
video_start = float(output_video.get("start_time", 0.0))
|
| 137 |
+
audio_start = float(output_audio.get("start_time", 0.0))
|
| 138 |
+
output_duration = float(output_probe["format"].get("duration", 0.0))
|
| 139 |
+
frame_tolerance = 1.0 / float(args.expected_fps) + 1e-3
|
| 140 |
+
if abs(video_start) > frame_tolerance or abs(audio_start) > frame_tolerance:
|
| 141 |
+
raise RuntimeError(
|
| 142 |
+
f"Muxed streams do not start together: video={video_start}, audio={audio_start}"
|
| 143 |
+
)
|
| 144 |
+
if output_duration + frame_tolerance < video_duration:
|
| 145 |
+
raise RuntimeError(
|
| 146 |
+
f"Mux truncated the motion video: input={video_duration}, output={output_duration}"
|
| 147 |
+
)
|
| 148 |
+
|
| 149 |
+
metadata["mux"] = {
|
| 150 |
+
"video_without_audio": str(video),
|
| 151 |
+
"video_with_audio": str(output),
|
| 152 |
+
"audio": str(audio),
|
| 153 |
+
"video_fps": video_fps,
|
| 154 |
+
"video_start_seconds": video_start,
|
| 155 |
+
"audio_start_seconds": audio_start,
|
| 156 |
+
"video_duration_seconds": video_duration,
|
| 157 |
+
"muxed_duration_seconds": output_duration,
|
| 158 |
+
"audio_mode": (
|
| 159 |
+
"complete_stream_timeline"
|
| 160 |
+
if bool(args.audio_is_timeline)
|
| 161 |
+
else "speech_only_legacy"
|
| 162 |
+
),
|
| 163 |
+
"audio_padding": (
|
| 164 |
+
"timeline_apad_only_if_container_rounding_requires_it"
|
| 165 |
+
if bool(args.audio_is_timeline)
|
| 166 |
+
else "silence_to_video_end"
|
| 167 |
+
),
|
| 168 |
+
"speech_delay_seconds": requested_speech_delay_seconds,
|
| 169 |
+
"applied_audio_delay_seconds": applied_audio_delay_seconds,
|
| 170 |
+
"motion_start_frame_50fps": metadata.get("motion_start_frame_50fps"),
|
| 171 |
+
"motion_start_seconds": metadata.get("motion_start_seconds"),
|
| 172 |
+
}
|
| 173 |
+
meta_path.write_text(
|
| 174 |
+
json.dumps(metadata, ensure_ascii=False, indent=2) + "\n", encoding="utf-8"
|
| 175 |
+
)
|
| 176 |
+
print(f"[done] synchronized video: {output}", flush=True)
|
| 177 |
+
|
| 178 |
+
|
| 179 |
+
if __name__ == "__main__":
|
| 180 |
+
main()
|
scripts/infer/render_third_party_review.py
ADDED
|
@@ -0,0 +1,287 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#!/usr/bin/env python3
|
| 2 |
+
"""Render a stratified final-model review from saved predictions, CPU only."""
|
| 3 |
+
from __future__ import annotations
|
| 4 |
+
|
| 5 |
+
import argparse
|
| 6 |
+
from concurrent.futures import ThreadPoolExecutor, as_completed
|
| 7 |
+
import hashlib
|
| 8 |
+
import json
|
| 9 |
+
import os
|
| 10 |
+
from pathlib import Path
|
| 11 |
+
import random
|
| 12 |
+
import shutil
|
| 13 |
+
import subprocess
|
| 14 |
+
import sys
|
| 15 |
+
import time
|
| 16 |
+
import wave
|
| 17 |
+
|
| 18 |
+
PROJECT = Path(__file__).resolve().parents[2]
|
| 19 |
+
sys.path[:0] = [str(PROJECT), str(PROJECT / "src")]
|
| 20 |
+
PROFILES = ("single_short", "multi_single", "multi_babble", "mixed_short", "continuous", "same_voice_aside")
|
| 21 |
+
LABELS = dict(zip(PROFILES, ("单段短人声", "多段单人声", "多段多人杂谈", "单/多人混合短声", "连续背景说话", "指令同音色旁话")))
|
| 22 |
+
ASSETS = ("motion.npy", "body_pos.csv", "body_quat.csv", "joint_pos.csv", "body_lin_vel.csv",
|
| 23 |
+
"body_ang_vel.csv", "joint_vel.csv", "metadata.txt", "infer_meta.json", "metrics.json", "session_recipe.json")
|
| 24 |
+
|
| 25 |
+
|
| 26 |
+
def read(path):
|
| 27 |
+
return json.loads(Path(path).read_text())
|
| 28 |
+
|
| 29 |
+
|
| 30 |
+
def digest(path):
|
| 31 |
+
h = hashlib.sha256()
|
| 32 |
+
with Path(path).open("rb") as handle:
|
| 33 |
+
for block in iter(lambda: handle.read(1024 * 1024), b""):
|
| 34 |
+
h.update(block)
|
| 35 |
+
return h.hexdigest()
|
| 36 |
+
|
| 37 |
+
|
| 38 |
+
def write(path, value):
|
| 39 |
+
path = Path(path)
|
| 40 |
+
temp = path.with_suffix(path.suffix + ".tmp")
|
| 41 |
+
temp.write_text(json.dumps(value, ensure_ascii=False, indent=2, allow_nan=False) + "\n")
|
| 42 |
+
temp.replace(path)
|
| 43 |
+
|
| 44 |
+
|
| 45 |
+
def select_samples(selection, report, seed=26100801):
|
| 46 |
+
rows, samples = selection["sessions"], report["samples"]
|
| 47 |
+
ids = [r["session_id"] for r in rows]
|
| 48 |
+
sample_ids = [r["session_id"] for r in samples]
|
| 49 |
+
if (report.get("status") != "complete" or report.get("evaluation_split") != "train_only"
|
| 50 |
+
or report.get("count") != 300 or len(rows) != 300 or len(samples) != 300
|
| 51 |
+
or len(set(ids)) != 300 or len(set(sample_ids)) != 300
|
| 52 |
+
or set(ids) != set(sample_ids) or selection["session_ids"] != ids):
|
| 53 |
+
raise ValueError("Expected the same complete unique train300 selection and report")
|
| 54 |
+
by_id = {r["session_id"]: r for r in rows}
|
| 55 |
+
for sample in samples:
|
| 56 |
+
row = by_id[sample["session_id"]]
|
| 57 |
+
if (sample["profile"] != row["session_type"]
|
| 58 |
+
or sample["background_language"] != row["background_language"]):
|
| 59 |
+
raise ValueError("Prediction profile/language differs from frozen selection")
|
| 60 |
+
rng, chosen = random.Random(seed), []
|
| 61 |
+
for profile in PROFILES:
|
| 62 |
+
for language in ("zh", "en"):
|
| 63 |
+
for high in (False, True):
|
| 64 |
+
pool = sorted((s for s in samples if s["profile"] == profile
|
| 65 |
+
and s["background_language"] == language
|
| 66 |
+
and by_id[s["session_id"]]["background_high"] == high), key=lambda s: s["session_id"])
|
| 67 |
+
if not pool:
|
| 68 |
+
raise ValueError(f"Empty review cell: {profile}/{language}/{high}")
|
| 69 |
+
chosen.append({**rng.choice(pool), "number": f"{len(chosen) + 1:02d}", "background_high": high})
|
| 70 |
+
return chosen
|
| 71 |
+
|
| 72 |
+
|
| 73 |
+
def contained(path, root):
|
| 74 |
+
path, root = Path(path).resolve(strict=True), Path(root).resolve(strict=True)
|
| 75 |
+
if path == root or not path.is_relative_to(root):
|
| 76 |
+
raise ValueError(f"Path escaped expected input root: {path}")
|
| 77 |
+
return path
|
| 78 |
+
|
| 79 |
+
|
| 80 |
+
def render_session_alive(pid):
|
| 81 |
+
"""Include orphaned encoders/renderers, not only the coordinating worker."""
|
| 82 |
+
for stat in Path("/proc").glob("[0-9]*/stat"):
|
| 83 |
+
try:
|
| 84 |
+
fields = stat.read_text().rsplit(")", 1)[1].split()
|
| 85 |
+
except (FileNotFoundError, ProcessLookupError):
|
| 86 |
+
continue
|
| 87 |
+
if fields[0] != "Z" and int(fields[3]) == int(pid):
|
| 88 |
+
return True
|
| 89 |
+
return False
|
| 90 |
+
|
| 91 |
+
|
| 92 |
+
def prepare(run, manifest):
|
| 93 |
+
import numpy as np
|
| 94 |
+
run = contained(run, PROJECT / "runs")
|
| 95 |
+
report_path = run / "final_train300/report.json"
|
| 96 |
+
report, result, selection = read(report_path), read(run / "result.json"), read(manifest)
|
| 97 |
+
if result["status"] != "complete" or report["step"] != result["step"]:
|
| 98 |
+
raise ValueError("Final report does not describe the completed run")
|
| 99 |
+
status = read(run / "run_status.json")
|
| 100 |
+
if digest(manifest) != status["manifest_sha256"]:
|
| 101 |
+
raise ValueError("Frozen train300 selection hash changed")
|
| 102 |
+
chosen = select_samples(selection, report)
|
| 103 |
+
for row in chosen:
|
| 104 |
+
source = contained(row["prediction_directory"], run / "final_train300/samples")
|
| 105 |
+
meta = read(source / "infer_meta.json")
|
| 106 |
+
recipe = read(source / "session_recipe.json")
|
| 107 |
+
if (meta["session_id"] != row["session_id"] or meta["global_step"] != report["step"]
|
| 108 |
+
or meta["artifact_role"] != "model_prediction_not_GT"
|
| 109 |
+
or recipe["split"] != "train" or recipe["session_id"] != row["session_id"]):
|
| 110 |
+
raise ValueError("Prediction provenance is inconsistent")
|
| 111 |
+
audio = contained(meta["frozen_audio_path"], PROJECT / "data/hfd_third_party_multi_v5/production/audio/train")
|
| 112 |
+
if digest(audio) != meta["frozen_audio_sha256"] or meta["frozen_audio_sha256"] != recipe["audio"]["sha256"]:
|
| 113 |
+
raise ValueError("Frozen mixed audio changed")
|
| 114 |
+
with wave.open(str(audio), "rb") as wav:
|
| 115 |
+
if (wav.getframerate(), wav.getnchannels(), wav.getsampwidth()) != (16000, 1, 2):
|
| 116 |
+
raise ValueError("Expected the original mono 16 kHz PCM16")
|
| 117 |
+
frames, seconds = round(wav.getnframes() * 50 / 16000), wav.getnframes() / 16000
|
| 118 |
+
motion = np.load(source / "motion.npy", allow_pickle=False)
|
| 119 |
+
if motion.shape != (frames, 36) or not np.isfinite(motion).all() or digest(source / "motion.npy") != row["motion_sha256"]:
|
| 120 |
+
raise ValueError("Saved predicted trajectory is incomplete or changed")
|
| 121 |
+
for name, width in (("body_pos.csv", 3), ("body_quat.csv", 4), ("joint_pos.csv", 29)):
|
| 122 |
+
values = np.loadtxt(source / name, delimiter=",", skiprows=1, ndmin=2)
|
| 123 |
+
if values.shape != (frames, width) or not np.isfinite(values).all():
|
| 124 |
+
raise ValueError(f"Render CSV has invalid length or values: {name}")
|
| 125 |
+
row.update(source_directory=str(source), audio_path=str(audio), audio_sha256=meta["frozen_audio_sha256"],
|
| 126 |
+
duration_seconds=seconds, source_files={name: digest(source / name) for name in ASSETS})
|
| 127 |
+
return {"schema": 1, "run_dir": str(run), "step": report["step"], "artifact_role": "model_prediction_not_GT",
|
| 128 |
+
"evaluation_split": "train_only", "seed": 26100801, "count": 24, "inference_repeated": False,
|
| 129 |
+
"report_sha256": digest(report_path), "manifest_sha256": digest(manifest),
|
| 130 |
+
"selection_policy": "one random sample per profile/language/loudness cell; never select by score",
|
| 131 |
+
"render": {"backend": "mesa_cpu", "workers": 4, "threads_each": 4, "fps": 25, "width": 320, "height": 240},
|
| 132 |
+
"audio_policy": "original mixed timeline; no gain/offset change; AAC encoding in MP4",
|
| 133 |
+
"samples": chosen}
|
| 134 |
+
|
| 135 |
+
|
| 136 |
+
def render_one(row, output):
|
| 137 |
+
from scripts.infer.third_party_checkpoint_probe import render_prediction
|
| 138 |
+
from scripts.infer.mux_full_duplex_audio import probe, stream, fraction
|
| 139 |
+
started = time.monotonic()
|
| 140 |
+
source, audio = Path(row["source_directory"]), Path(row["audio_path"])
|
| 141 |
+
destination = output / "samples" / row["number"]
|
| 142 |
+
flat = output / f"{row['number']}.mp4"
|
| 143 |
+
saved_result = destination / "render_result.json"
|
| 144 |
+
for path in (output, output / "samples", destination, flat, saved_result, *(destination / name for name in ASSETS)):
|
| 145 |
+
if path.is_symlink():
|
| 146 |
+
raise ValueError("Refusing symlink in dedicated preview outputs")
|
| 147 |
+
if saved_result.exists():
|
| 148 |
+
result = read(saved_result)
|
| 149 |
+
if (result.get("status") != "complete" or any(result.get(key) != value for key, value in row.items())
|
| 150 |
+
or not result.get("full_decode_passed") or not result.get("source_files_unchanged")
|
| 151 |
+
or result.get("video") != str(flat) or digest(flat) != result["video_sha256"]
|
| 152 |
+
or flat.stat().st_size != result.get("video_bytes")):
|
| 153 |
+
raise ValueError("Completed preview does not match the frozen render plan")
|
| 154 |
+
if any(digest(source / name) != expected for name, expected in row["source_files"].items()) or digest(audio) != row["audio_sha256"]:
|
| 155 |
+
raise ValueError("Original prediction/audio changed before preview reuse")
|
| 156 |
+
return {**result, "reused_existing_video": True}
|
| 157 |
+
if flat.exists():
|
| 158 |
+
raise ValueError("Unverified video exists without a completed render result; refusing overwrite")
|
| 159 |
+
# Only unfinished copies in this dedicated output may be rewritten on resume.
|
| 160 |
+
destination.mkdir(parents=True, exist_ok=True)
|
| 161 |
+
for name, expected in row["source_files"].items():
|
| 162 |
+
if digest(source / name) != expected:
|
| 163 |
+
raise ValueError("Source prediction changed after preflight")
|
| 164 |
+
shutil.copy2(source / name, destination / name)
|
| 165 |
+
if digest(destination / name) != expected:
|
| 166 |
+
raise ValueError("Prediction copy changed")
|
| 167 |
+
if digest(audio) != row["audio_sha256"]:
|
| 168 |
+
raise ValueError("Audio changed after preflight")
|
| 169 |
+
video = render_prediction(destination, audio)
|
| 170 |
+
media = probe(video); v, a = stream(media, "video"), stream(media, "audio")
|
| 171 |
+
if (v["width"], v["height"]) != (320, 240) or abs(fraction(v["avg_frame_rate"]) - 25) > .01:
|
| 172 |
+
raise ValueError("Rendered dimensions/FPS changed")
|
| 173 |
+
if int(a["channels"]) != 1 or int(a["sample_rate"]) != 16000:
|
| 174 |
+
raise ValueError("Rendered audio stream format changed")
|
| 175 |
+
if float(media["format"]["duration"]) < row["duration_seconds"] - .041:
|
| 176 |
+
raise ValueError("Rendered video truncated the input timeline")
|
| 177 |
+
for name, expected in row["source_files"].items():
|
| 178 |
+
if digest(source / name) != expected:
|
| 179 |
+
raise ValueError("Original prediction was modified during rendering")
|
| 180 |
+
if digest(audio) != row["audio_sha256"]:
|
| 181 |
+
raise ValueError("Original audio was modified during rendering")
|
| 182 |
+
# A second name for the same local output; no duplicate video storage.
|
| 183 |
+
os.link(video, flat)
|
| 184 |
+
result = {**row, "status": "complete", "video": str(flat), "video_sha256": digest(flat),
|
| 185 |
+
"video_bytes": flat.stat().st_size, "full_decode_passed": True, "source_files_unchanged": True,
|
| 186 |
+
"render_seconds": time.monotonic() - started}
|
| 187 |
+
write(destination / "render_result.json", result)
|
| 188 |
+
return result
|
| 189 |
+
|
| 190 |
+
|
| 191 |
+
def publish(output, plan, completed):
|
| 192 |
+
lines = ["# 最终模型预测视频(不是 GT)", "", f"Checkpoint **{plan['step']}**;已完成 **{len(completed)}/24**。",
|
| 193 |
+
"", "复用已保存的训练集预测,不重复推理;六类×中英文背景×普通/较高背景音量,每格固定随机抽一条。",
|
| 194 |
+
"320×240、25fps审核预览;原始预测仍为50Hz。完整原混音不调音量、不改时间位置,MP4音轨编码为AAC。",
|
| 195 |
+
"", "下表指标对应各自单条样本,仅供定位问题,不作为总体效果估计。", "",
|
| 196 |
+
"| 序号/视频 | 类型 | 背景语言 | 背景音量 | 时长 | 控制准确率 | 背景误动率 |",
|
| 197 |
+
"|---|---|---|---|---:|---:|---:|"]
|
| 198 |
+
by_number = {r["number"]: r for r in completed}
|
| 199 |
+
for row in plan["samples"]:
|
| 200 |
+
number = row["number"]; label = f"[{number}]({number}.mp4)" if number in by_number else f"{number} 待完成"
|
| 201 |
+
metrics = row["metrics"]; bg = metrics.get("background_only_false_motion_rate")
|
| 202 |
+
bg = "无可评分区间" if bg is None else f"{bg:.1%}"
|
| 203 |
+
lines.append(f"| {label} | {LABELS[row['profile']]} | {'中文' if row['background_language']=='zh' else '英文'} | "
|
| 204 |
+
f"{'较高' if row['background_high'] else '普通'} | {row['duration_seconds']:.1f}s | {metrics['control_accuracy']:.1%} | {bg} |")
|
| 205 |
+
lines.extend(["", "[原始逐条评估](../final_train300/report.json) · [抽样与来源校验](plan.json) · [渲染状态](status.json)", ""])
|
| 206 |
+
(output / "README.md").write_text("\n".join(lines))
|
| 207 |
+
run = Path(plan["run_dir"]); review = run / "review.md"
|
| 208 |
+
if review.exists():
|
| 209 |
+
marker = "<!-- final_prediction_videos -->"
|
| 210 |
+
text = review.read_text().split(marker)[0].rstrip()
|
| 211 |
+
text = text.replace("| — | 尚无完整 probe 记录 | — | — | — | — |",
|
| 212 |
+
"| — | 本轮采用轮末数值评估,见下方逐轮表 | — | — | — | [预测视频](prediction_videos/README.md) |")
|
| 213 |
+
review.write_text(text + f"\n\n{marker}\n## 最终模型预测视频\n\n"
|
| 214 |
+
f"Checkpoint {plan['step']},已渲染 {len(completed)}/24 条,**模型预测,不是GT**。\n\n"
|
| 215 |
+
"[01–24 视频与类型表](prediction_videos/README.md) · [最终300条评估](final_train300/report.json)\n")
|
| 216 |
+
|
| 217 |
+
|
| 218 |
+
def worker(output):
|
| 219 |
+
# Imports occur before parallel rendering; no CUDA model is loaded.
|
| 220 |
+
from scripts.infer.third_party_checkpoint_probe import check_renderer
|
| 221 |
+
check_renderer()
|
| 222 |
+
plan = read(output / "plan.json")
|
| 223 |
+
completed, failures = [], []
|
| 224 |
+
started = time.time()
|
| 225 |
+
def status(phase):
|
| 226 |
+
write(output / "status.json", {"phase": phase, "pid": os.getpid(), "started_at": started,
|
| 227 |
+
"updated_at": time.time(), "completed": len(completed), "total": 24, "failed": len(failures),
|
| 228 |
+
"samples": completed, "errors": failures, "gpu_used": False, "inference_repeated": False,
|
| 229 |
+
"workers": plan["render"]["workers"]})
|
| 230 |
+
publish(output, plan, completed)
|
| 231 |
+
status("rendering")
|
| 232 |
+
with ThreadPoolExecutor(max_workers=plan["render"]["workers"]) as pool:
|
| 233 |
+
jobs = {pool.submit(render_one, row, output): row for row in plan["samples"]}
|
| 234 |
+
for future in as_completed(jobs):
|
| 235 |
+
row = jobs[future]
|
| 236 |
+
try:
|
| 237 |
+
result = future.result(); completed.append(result)
|
| 238 |
+
print(json.dumps({"number": row["number"], "seconds": result["render_seconds"], "status": "complete"}), flush=True)
|
| 239 |
+
except Exception as error:
|
| 240 |
+
failures.append({"number": row["number"], "error": repr(error)})
|
| 241 |
+
print(json.dumps(failures[-1]), flush=True)
|
| 242 |
+
status("rendering")
|
| 243 |
+
status("failed" if failures else "completed")
|
| 244 |
+
if failures:
|
| 245 |
+
raise RuntimeError(f"{len(failures)} preview renders failed; inspect per-sample render.log")
|
| 246 |
+
|
| 247 |
+
|
| 248 |
+
def main():
|
| 249 |
+
parser = argparse.ArgumentParser(description=__doc__)
|
| 250 |
+
parser.add_argument("--run-dir", type=Path, required=True)
|
| 251 |
+
parser.add_argument("--manifest", type=Path, default=PROJECT / "data/hfd_third_party_multi_v5/training_pilot_300/selection.json")
|
| 252 |
+
parser.add_argument("--workers", type=int, choices=range(1, 17), default=12)
|
| 253 |
+
parser.add_argument("--resume", action="store_true", help="Reuse verified videos after the previous render worker has exited")
|
| 254 |
+
mode = parser.add_mutually_exclusive_group(required=True)
|
| 255 |
+
for name in ("launch", "worker", "check-only"):
|
| 256 |
+
mode.add_argument(f"--{name}", action="store_true")
|
| 257 |
+
args = parser.parse_args(); run = contained(args.run_dir, PROJECT / "runs"); output = run / "prediction_videos"
|
| 258 |
+
if args.resume and not args.launch:
|
| 259 |
+
parser.error("--resume requires --launch")
|
| 260 |
+
os.environ.update(CUDA_VISIBLE_DEVICES="", PYTHONDONTWRITEBYTECODE="1", OMP_NUM_THREADS="1", MKL_NUM_THREADS="1")
|
| 261 |
+
if args.worker:
|
| 262 |
+
worker(output); return
|
| 263 |
+
plan = prepare(run, args.manifest)
|
| 264 |
+
plan["render"]["workers"] = args.workers
|
| 265 |
+
if args.check_only:
|
| 266 |
+
print(json.dumps({"passed": True, "step": plan["step"], "count": 24, "seconds": sum(r['duration_seconds'] for r in plan['samples'])})); return
|
| 267 |
+
if args.resume:
|
| 268 |
+
previous = read(output / "plan.json")
|
| 269 |
+
if any(previous[key] != plan[key] for key in plan if key != "render"):
|
| 270 |
+
raise ValueError("Cannot resume a different frozen prediction review")
|
| 271 |
+
if any(previous["render"][key] != plan["render"][key] for key in plan["render"] if key != "workers"):
|
| 272 |
+
raise ValueError("Resume can change worker count only, not rendering settings")
|
| 273 |
+
previous_status = read(output / "status.json")
|
| 274 |
+
if render_session_alive(previous_status["pid"]):
|
| 275 |
+
raise ValueError("Previous render session is still present; refusing concurrent launch")
|
| 276 |
+
else:
|
| 277 |
+
output.mkdir(exist_ok=False)
|
| 278 |
+
write(output / "plan.json", plan)
|
| 279 |
+
with (output / "render.log").open("a" if args.resume else "x") as log:
|
| 280 |
+
child = subprocess.Popen([sys.executable, str(Path(__file__).resolve()), "--run-dir", str(run), "--worker"],
|
| 281 |
+
cwd=PROJECT, env=os.environ.copy(), stdin=subprocess.DEVNULL, stdout=log, stderr=subprocess.STDOUT,
|
| 282 |
+
start_new_session=True)
|
| 283 |
+
print(json.dumps({"render_pid": child.pid, "output": str(output), "index": str(output / 'README.md')}))
|
| 284 |
+
|
| 285 |
+
|
| 286 |
+
if __name__ == "__main__":
|
| 287 |
+
main()
|
scripts/infer/render_third_party_scale.py
ADDED
|
@@ -0,0 +1,151 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#!/usr/bin/env python3
|
| 2 |
+
"""CPU-only final scale previews, with separate train and held-out indexes."""
|
| 3 |
+
from __future__ import annotations
|
| 4 |
+
|
| 5 |
+
import argparse
|
| 6 |
+
from concurrent.futures import ThreadPoolExecutor, as_completed
|
| 7 |
+
import os
|
| 8 |
+
from pathlib import Path
|
| 9 |
+
import sys
|
| 10 |
+
import time
|
| 11 |
+
import wave
|
| 12 |
+
|
| 13 |
+
PROJECT = Path(__file__).resolve().parents[2]
|
| 14 |
+
sys.path[:0] = [str(PROJECT), str(PROJECT / "src")]
|
| 15 |
+
from scripts.infer.render_third_party_review import (
|
| 16 |
+
ASSETS, LABELS, contained, digest, read, render_one, write)
|
| 17 |
+
|
| 18 |
+
|
| 19 |
+
def selected_rows(selection, report, step):
|
| 20 |
+
if (selection.get("scaleup") is not True or selection.get("split") not in ("train", "validation")
|
| 21 |
+
or report.get("status") != "complete" or report.get("step") != step
|
| 22 |
+
or report.get("evaluation_split") != selection["split"]):
|
| 23 |
+
raise ValueError("Final prediction split/step differs from its frozen selection")
|
| 24 |
+
by_id = {r["session_id"]: r for r in selection["sessions"]}
|
| 25 |
+
samples = {r["session_id"]: r for r in report["samples"]}
|
| 26 |
+
if len(samples) != report["count"] or len(samples) != len(report["samples"]):
|
| 27 |
+
raise ValueError("Duplicate/incomplete final predictions")
|
| 28 |
+
ids = selection["video_session_ids"]
|
| 29 |
+
if not ids or len(ids) != len(set(ids)) or not set(ids) <= samples.keys() & by_id.keys():
|
| 30 |
+
raise ValueError("Missing predetermined video samples")
|
| 31 |
+
rows = []
|
| 32 |
+
for index, sid in enumerate(ids, 1):
|
| 33 |
+
row, chosen = samples[sid], by_id[sid]
|
| 34 |
+
if (row["profile"] != chosen["session_type"] or row["split"] != selection["split"]
|
| 35 |
+
or row["background_language"] != chosen["background_language"]):
|
| 36 |
+
raise ValueError("Prediction labels differ from frozen selection")
|
| 37 |
+
rows.append({**row, "number": f"{index:02d}", "background_high": chosen["background_high"]})
|
| 38 |
+
return rows
|
| 39 |
+
|
| 40 |
+
|
| 41 |
+
def prepare(run, manifest):
|
| 42 |
+
import numpy as np
|
| 43 |
+
run = contained(run, PROJECT / "runs")
|
| 44 |
+
result, identity = read(run / "result.json"), read(run / "scale_identity.json")
|
| 45 |
+
if result["status"] != "complete":
|
| 46 |
+
raise ValueError("Training/evaluation has not completed")
|
| 47 |
+
rows, hashes = [], {}
|
| 48 |
+
for split in ("train", "validation"):
|
| 49 |
+
path = manifest if split == "train" else manifest.parent / "validation.json"
|
| 50 |
+
key = "manifest_sha256" if split == "train" else "validation_sha256"
|
| 51 |
+
if digest(path) != identity[key]:
|
| 52 |
+
raise ValueError("Frozen video selection changed")
|
| 53 |
+
selection = read(path)
|
| 54 |
+
report_path = run / "evaluations/final" / split / "report.json"
|
| 55 |
+
report = read(report_path); hashes[split] = digest(report_path)
|
| 56 |
+
for row in selected_rows(selection, report, result["step"]):
|
| 57 |
+
source = contained(row["prediction_directory"], report_path.parent / "samples")
|
| 58 |
+
meta, recipe = read(source / "infer_meta.json"), read(source / "session_recipe.json")
|
| 59 |
+
if (meta["session_id"] != row["session_id"] or meta["global_step"] != result["step"]
|
| 60 |
+
or meta.get("evaluation_split") != split or meta["artifact_role"] != "model_prediction_not_GT"
|
| 61 |
+
or recipe["session_id"] != row["session_id"] or recipe["split"] != split):
|
| 62 |
+
raise ValueError("Prediction provenance mismatch")
|
| 63 |
+
audio = contained(meta["frozen_audio_path"], Path(selection["production_root"]) / "audio" / split)
|
| 64 |
+
if digest(audio) != meta["frozen_audio_sha256"] or digest(audio) != recipe["audio"]["sha256"]:
|
| 65 |
+
raise ValueError("Original mixed audio changed")
|
| 66 |
+
with wave.open(str(audio), "rb") as wav:
|
| 67 |
+
if (wav.getframerate(), wav.getnchannels(), wav.getsampwidth()) != (16000, 1, 2):
|
| 68 |
+
raise ValueError("Expected original mono 16kHz PCM16")
|
| 69 |
+
seconds = wav.getnframes() / 16000
|
| 70 |
+
frames = round(seconds * 50)
|
| 71 |
+
motion = np.load(source / "motion.npy", allow_pickle=False)
|
| 72 |
+
if motion.shape != (frames, 36) or not np.isfinite(motion).all() or digest(source / "motion.npy") != row["motion_sha256"]:
|
| 73 |
+
raise ValueError("Prediction trajectory changed or incomplete")
|
| 74 |
+
for name, width in (("body_pos.csv", 3), ("body_quat.csv", 4), ("joint_pos.csv", 29)):
|
| 75 |
+
values = np.loadtxt(source / name, delimiter=",", skiprows=1, ndmin=2)
|
| 76 |
+
if values.shape != (frames, width) or not np.isfinite(values).all():
|
| 77 |
+
raise ValueError(f"Invalid predicted render CSV: {name}")
|
| 78 |
+
rows.append({**row, "source_directory": str(source), "audio_path": str(audio),
|
| 79 |
+
"audio_sha256": meta["frozen_audio_sha256"], "duration_seconds": seconds,
|
| 80 |
+
"source_files": {name: digest(source / name) for name in ASSETS}})
|
| 81 |
+
return {"schema": 1, "run_dir": str(run), "step": result["step"], "count": len(rows), "samples": rows,
|
| 82 |
+
"report_hashes": hashes, "gpu_used": False, "inference_repeated": False,
|
| 83 |
+
"selection_policy": "predetermined one per available class/language/loudness cell, never by model score",
|
| 84 |
+
"artifact_role": "model_prediction_not_GT", "render": {"workers": 12, "fps": 25, "size": [320, 240]}}
|
| 85 |
+
|
| 86 |
+
|
| 87 |
+
def publish(output, plan, completed):
|
| 88 |
+
done = {(r["split"], r["number"]) for r in completed}
|
| 89 |
+
lines = ["# 全量训练最终预测视频(不是 GT)", "",
|
| 90 |
+
f"新阶段 checkpoint {plan['step']};完成 {len(completed)}/{plan['count']}。",
|
| 91 |
+
"复用完整预测;原混音不改变音量和时间。CPU 并行渲染,320×240、25fps;原预测仍为50Hz。",
|
| 92 |
+
"样例在训练前按六类×中英文背景×普通/较高音量固定随机选定,不按效果挑选。", ""]
|
| 93 |
+
for split, label in (("train", "训练集(拟合)"), ("validation", "独立验证集(未参与优化)")):
|
| 94 |
+
lines += [f"## {label}", "", "| 视频序号 | 类型 | 背景语言 | 背景音量 | 控制准确率 | 背景误动率 |",
|
| 95 |
+
"|---|---|---|---|---:|---:|"]
|
| 96 |
+
for row in plan["samples"]:
|
| 97 |
+
if row["split"] != split:
|
| 98 |
+
continue
|
| 99 |
+
number = row["number"]
|
| 100 |
+
link = f"[{number}]({split}/{number}.mp4)" if (split, number) in done else f"{number} 待生成"
|
| 101 |
+
metrics = row["metrics"]; bg = metrics.get("background_only_false_motion_rate")
|
| 102 |
+
bg_text = "无评分区间" if bg is None else f"{bg:.1%}"
|
| 103 |
+
lines.append(f"| {link} | {LABELS[row['profile']]} | {'中文' if row['background_language']=='zh' else '英文'} | "
|
| 104 |
+
f"{'较高' if row['background_high'] else '普通'} | {metrics['control_accuracy']:.1%} | {bg_text} |")
|
| 105 |
+
lines += ["", f"[本组原始逐条指标](../evaluations/final/{split}/report.json)", ""]
|
| 106 |
+
lines += ["单条视频指标用于定位问题,不代替全部验证结果,也不代表机器人实际执行成功。", "",
|
| 107 |
+
"[渲染状态](status.json) · [完整抽样溯源](plan.json)", ""]
|
| 108 |
+
(output / "README.md").write_text("\n".join(lines))
|
| 109 |
+
|
| 110 |
+
|
| 111 |
+
def main():
|
| 112 |
+
parser = argparse.ArgumentParser(description=__doc__)
|
| 113 |
+
parser.add_argument("--run-dir", type=Path, required=True)
|
| 114 |
+
parser.add_argument("--manifest", type=Path, required=True)
|
| 115 |
+
args = parser.parse_args()
|
| 116 |
+
os.environ.update(CUDA_VISIBLE_DEVICES="", OMP_NUM_THREADS="1", MKL_NUM_THREADS="1", PYTHONDONTWRITEBYTECODE="1")
|
| 117 |
+
from scripts.infer.third_party_checkpoint_probe import check_renderer
|
| 118 |
+
plan = prepare(args.run_dir, args.manifest.resolve())
|
| 119 |
+
output = Path(plan["run_dir"]) / "prediction_videos"
|
| 120 |
+
output.mkdir(exist_ok=False)
|
| 121 |
+
write(output / "plan.json", plan)
|
| 122 |
+
for split in ("train", "validation"):
|
| 123 |
+
(output / split).mkdir()
|
| 124 |
+
completed, failures = [], []
|
| 125 |
+
def status(phase):
|
| 126 |
+
write(output / "status.json", {"phase": phase, "time": time.time(), "pid": os.getpid(),
|
| 127 |
+
"completed": len(completed), "total": plan["count"], "failures": failures,
|
| 128 |
+
"gpu_used": False, "inference_repeated": False})
|
| 129 |
+
publish(output, plan, completed)
|
| 130 |
+
status("starting")
|
| 131 |
+
try:
|
| 132 |
+
check_renderer()
|
| 133 |
+
with ThreadPoolExecutor(max_workers=12) as pool:
|
| 134 |
+
jobs = {pool.submit(render_one, row, output / row["split"]): row for row in plan["samples"]}
|
| 135 |
+
for future in as_completed(jobs):
|
| 136 |
+
row = jobs[future]
|
| 137 |
+
try:
|
| 138 |
+
completed.append(future.result())
|
| 139 |
+
print(f"{row['split']}/{row['number']} completed", flush=True)
|
| 140 |
+
except Exception as error:
|
| 141 |
+
failures.append({"split": row["split"], "number": row["number"], "error": repr(error)})
|
| 142 |
+
status("rendering")
|
| 143 |
+
if failures:
|
| 144 |
+
raise RuntimeError("Some final videos failed; inspect status.json and per-sample render.log")
|
| 145 |
+
status("completed")
|
| 146 |
+
except Exception as error:
|
| 147 |
+
failures.append({"error": repr(error)}); status("failed"); raise
|
| 148 |
+
|
| 149 |
+
|
| 150 |
+
if __name__ == "__main__":
|
| 151 |
+
main()
|
scripts/infer/run_multiturn_infer_suite.py
ADDED
|
@@ -0,0 +1,700 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#!/usr/bin/env python3
|
| 2 |
+
"""Run HFD multi-turn evaluation cases through a dynamic GPU queue."""
|
| 3 |
+
|
| 4 |
+
from __future__ import annotations
|
| 5 |
+
|
| 6 |
+
import hashlib
|
| 7 |
+
import json
|
| 8 |
+
import os
|
| 9 |
+
import re
|
| 10 |
+
import subprocess
|
| 11 |
+
import sys
|
| 12 |
+
import time
|
| 13 |
+
from collections import deque
|
| 14 |
+
from dataclasses import dataclass
|
| 15 |
+
from pathlib import Path
|
| 16 |
+
from typing import Any
|
| 17 |
+
|
| 18 |
+
|
| 19 |
+
PROJECT_ROOT = Path(__file__).resolve().parents[2]
|
| 20 |
+
DEFAULT_SUITE_V23 = PROJECT_ROOT / "scripts/infer/multiturn_auto_suite_v2.json"
|
| 21 |
+
DEFAULT_SUITE_V24 = PROJECT_ROOT / "scripts/infer/multiturn_auto_suite_v3.json"
|
| 22 |
+
|
| 23 |
+
|
| 24 |
+
@dataclass(frozen=True)
|
| 25 |
+
class InferenceCase:
|
| 26 |
+
name: str
|
| 27 |
+
interaction_mode: str
|
| 28 |
+
data_split: str
|
| 29 |
+
scenario_path: Path
|
| 30 |
+
scenario: dict[str, Any]
|
| 31 |
+
interaction_subtype: str = ""
|
| 32 |
+
|
| 33 |
+
|
| 34 |
+
def _truthy(value: str | None) -> bool:
|
| 35 |
+
return str(value or "").strip().lower() in {"1", "true", "yes", "on"}
|
| 36 |
+
|
| 37 |
+
|
| 38 |
+
def _motion_source_id(namespace: str, filename: str) -> str:
|
| 39 |
+
leaf = filename.replace("\\", "/").rstrip("/").rsplit("/", 1)[-1]
|
| 40 |
+
if leaf.lower().endswith(".csv"):
|
| 41 |
+
leaf = leaf[:-4]
|
| 42 |
+
return f"{namespace.strip().strip('/')}/{leaf}"
|
| 43 |
+
|
| 44 |
+
|
| 45 |
+
def _source_is_val(source_id: str, *, split_seed: int, val_ratio: float) -> bool:
|
| 46 |
+
digest = hashlib.sha1(
|
| 47 |
+
f"psi0-motion-source-split-v1\0{split_seed}\0{source_id}".encode()
|
| 48 |
+
).digest()
|
| 49 |
+
threshold = int(float(val_ratio) * (1 << 64))
|
| 50 |
+
return int.from_bytes(digest[:8], "big", signed=False) < threshold
|
| 51 |
+
|
| 52 |
+
|
| 53 |
+
def _load_json(path: Path) -> dict[str, Any]:
|
| 54 |
+
payload = json.loads(path.read_text(encoding="utf-8"))
|
| 55 |
+
if not isinstance(payload, dict):
|
| 56 |
+
raise ValueError(f"JSON root must be an object: {path}")
|
| 57 |
+
return payload
|
| 58 |
+
|
| 59 |
+
|
| 60 |
+
def validate_scenario(
|
| 61 |
+
path: Path,
|
| 62 |
+
*,
|
| 63 |
+
run_config: dict[str, Any],
|
| 64 |
+
expected_mode: str | None = None,
|
| 65 |
+
expected_split: str | None = None,
|
| 66 |
+
expected_subtype: str | None = None,
|
| 67 |
+
) -> InferenceCase:
|
| 68 |
+
payload = _load_json(path)
|
| 69 |
+
if payload.get("format") != "hfd_multiturn_auto_infer_scenario_v1":
|
| 70 |
+
raise ValueError(f"Unsupported multi-turn scenario: {path}")
|
| 71 |
+
turns = list(payload.get("turns", []))
|
| 72 |
+
if not 2 <= len(turns) <= 4:
|
| 73 |
+
raise ValueError(f"Scenario must contain 2-4 turns, got {len(turns)}")
|
| 74 |
+
mode = str(payload.get("interaction_mode", "turn-taking")).strip().lower()
|
| 75 |
+
supported_modes = {
|
| 76 |
+
"turn-taking",
|
| 77 |
+
"interruption",
|
| 78 |
+
"emergency-stop",
|
| 79 |
+
"safe-hold",
|
| 80 |
+
"backchannel",
|
| 81 |
+
"hesitation",
|
| 82 |
+
"pause",
|
| 83 |
+
}
|
| 84 |
+
if mode not in supported_modes:
|
| 85 |
+
raise ValueError(f"Unsupported interaction_mode={mode!r}: {path}")
|
| 86 |
+
split = str(payload.get("data_split", "custom")).strip().lower()
|
| 87 |
+
if split not in {"train", "val", "custom"}:
|
| 88 |
+
raise ValueError(f"Unsupported data_split={split!r}: {path}")
|
| 89 |
+
if expected_mode is not None and mode != expected_mode:
|
| 90 |
+
raise ValueError(f"Suite/scenario mode mismatch: {expected_mode} != {mode}")
|
| 91 |
+
if expected_split is not None and split != expected_split:
|
| 92 |
+
raise ValueError(f"Suite/scenario split mismatch: {expected_split} != {split}")
|
| 93 |
+
subtype = str(payload.get("interaction_subtype", "")).strip().lower()
|
| 94 |
+
valid_subtypes = {
|
| 95 |
+
"hesitation": {"filled-pause", "self-repair"},
|
| 96 |
+
"pause": {"silent-pause"},
|
| 97 |
+
}
|
| 98 |
+
if mode in valid_subtypes:
|
| 99 |
+
if subtype not in valid_subtypes[mode]:
|
| 100 |
+
raise ValueError(
|
| 101 |
+
f"Invalid {mode} interaction_subtype={subtype!r}: {path}"
|
| 102 |
+
)
|
| 103 |
+
elif subtype:
|
| 104 |
+
raise ValueError(
|
| 105 |
+
f"interaction_subtype is only valid for hesitation/pause: {path}"
|
| 106 |
+
)
|
| 107 |
+
if expected_subtype is not None and subtype != expected_subtype:
|
| 108 |
+
raise ValueError(
|
| 109 |
+
f"Suite/scenario subtype mismatch: {expected_subtype} != {subtype}"
|
| 110 |
+
)
|
| 111 |
+
|
| 112 |
+
gaps = [int(value) for value in payload["inter_turn_background_steps_by_turn"]]
|
| 113 |
+
if len(gaps) != len(turns) - 1 or any(value < 0 for value in gaps):
|
| 114 |
+
raise ValueError(f"Invalid per-turn background gaps: {gaps}")
|
| 115 |
+
declared_gap_seconds = [
|
| 116 |
+
float(value)
|
| 117 |
+
for value in payload.get("inter_turn_background_seconds_by_turn", [])
|
| 118 |
+
]
|
| 119 |
+
if declared_gap_seconds:
|
| 120 |
+
expected_gap_seconds = [0.4 * value for value in gaps]
|
| 121 |
+
if len(declared_gap_seconds) != len(gaps) or any(
|
| 122 |
+
abs(observed - expected) > 1e-9
|
| 123 |
+
for observed, expected in zip(
|
| 124 |
+
declared_gap_seconds,
|
| 125 |
+
expected_gap_seconds,
|
| 126 |
+
)
|
| 127 |
+
):
|
| 128 |
+
raise ValueError(
|
| 129 |
+
"Scenario gap seconds disagree with 0.4-second Qwen steps: "
|
| 130 |
+
f"{declared_gap_seconds} != {expected_gap_seconds}"
|
| 131 |
+
)
|
| 132 |
+
if mode in {"interruption", "emergency-stop", "safe-hold"}:
|
| 133 |
+
gap_seconds = [0.4 * value for value in gaps]
|
| 134 |
+
if any(value < 2.0 or value > 6.0 for value in gap_seconds):
|
| 135 |
+
raise ValueError(
|
| 136 |
+
"Mid-motion command gaps must stay within 2-6 seconds: "
|
| 137 |
+
f"{gap_seconds}"
|
| 138 |
+
)
|
| 139 |
+
if mode == "backchannel":
|
| 140 |
+
marked = [
|
| 141 |
+
index
|
| 142 |
+
for index, turn in enumerate(turns)
|
| 143 |
+
if str(turn.get("backchannel_intent", "")).strip()
|
| 144 |
+
]
|
| 145 |
+
if len(marked) != 1 or marked[0] == 0:
|
| 146 |
+
raise ValueError(
|
| 147 |
+
"Backchannel scenario needs exactly one acknowledgement after "
|
| 148 |
+
f"a real action command: indices={marked}"
|
| 149 |
+
)
|
| 150 |
+
if str(turns[marked[0]]["backchannel_intent"]).strip() not in {
|
| 151 |
+
"continuer",
|
| 152 |
+
"assessment",
|
| 153 |
+
}:
|
| 154 |
+
raise ValueError("Unknown backchannel intent category")
|
| 155 |
+
voice_stems = {
|
| 156 |
+
Path(str(turn["audio_path"])).stem for turn in turns
|
| 157 |
+
}
|
| 158 |
+
if len(voice_stems) != 1:
|
| 159 |
+
raise ValueError(
|
| 160 |
+
"Backchannel scenario must keep one speaker/language identity: "
|
| 161 |
+
f"{sorted(voice_stems)}"
|
| 162 |
+
)
|
| 163 |
+
onset_gap_seconds = 0.4 * gaps[marked[0] - 1]
|
| 164 |
+
if not 1.2 <= onset_gap_seconds <= 6.0:
|
| 165 |
+
raise ValueError(
|
| 166 |
+
"Backchannel must arrive during active motion: "
|
| 167 |
+
f"gap={onset_gap_seconds}s"
|
| 168 |
+
)
|
| 169 |
+
|
| 170 |
+
window_counts = [
|
| 171 |
+
int(value) for value in payload.get("turn_audio_window_counts", [])
|
| 172 |
+
]
|
| 173 |
+
expected_starts = [
|
| 174 |
+
int(value) for value in payload.get("expected_turn_window_starts", [])
|
| 175 |
+
]
|
| 176 |
+
if window_counts or expected_starts:
|
| 177 |
+
if len(window_counts) != len(turns) or len(expected_starts) != len(turns):
|
| 178 |
+
raise ValueError("Timeline audit needs one window count/start per turn")
|
| 179 |
+
derived_starts = [0]
|
| 180 |
+
for count, gap in zip(window_counts, gaps):
|
| 181 |
+
if count <= 0:
|
| 182 |
+
raise ValueError(f"Invalid audio window counts: {window_counts}")
|
| 183 |
+
derived_starts.append(derived_starts[-1] + count + gap)
|
| 184 |
+
if derived_starts != expected_starts:
|
| 185 |
+
raise ValueError(
|
| 186 |
+
f"Scenario gap timeline mismatch: {derived_starts} != {expected_starts}"
|
| 187 |
+
)
|
| 188 |
+
|
| 189 |
+
source_starts = [
|
| 190 |
+
int(value)
|
| 191 |
+
for value in payload.get("source_turn_speech_start_frames_50fps", [])
|
| 192 |
+
]
|
| 193 |
+
if source_starts:
|
| 194 |
+
if len(source_starts) != len(turns) or not expected_starts:
|
| 195 |
+
raise ValueError(
|
| 196 |
+
"Source timeline audit needs one source start and one expected "
|
| 197 |
+
"inference start per turn"
|
| 198 |
+
)
|
| 199 |
+
relative_frames = [value - source_starts[0] for value in source_starts]
|
| 200 |
+
if any(value % 20 != 0 for value in relative_frames):
|
| 201 |
+
raise ValueError(
|
| 202 |
+
"Source speech starts are not aligned to the 20-frame Qwen grid: "
|
| 203 |
+
f"{source_starts}"
|
| 204 |
+
)
|
| 205 |
+
source_relative_steps = [value // 20 for value in relative_frames]
|
| 206 |
+
if source_relative_steps != expected_starts:
|
| 207 |
+
raise ValueError(
|
| 208 |
+
"Scenario inference starts do not reproduce the source speech "
|
| 209 |
+
f"timeline: {source_relative_steps} != {expected_starts}"
|
| 210 |
+
)
|
| 211 |
+
|
| 212 |
+
data = dict(run_config.get("data", {}))
|
| 213 |
+
train = dict(run_config.get("train", {}))
|
| 214 |
+
active_semantics = int(train.get("full_duplex_semantics_version", 0))
|
| 215 |
+
required_semantics = int(payload.get("required_semantics_version", 0))
|
| 216 |
+
if active_semantics < required_semantics:
|
| 217 |
+
raise ValueError(
|
| 218 |
+
f"Scenario requires semantics {required_semantics}, run has {active_semantics}"
|
| 219 |
+
)
|
| 220 |
+
if mode == "interruption" and (
|
| 221 |
+
active_semantics < 20 or not bool(data.get("interruption_sessions", False))
|
| 222 |
+
):
|
| 223 |
+
raise ValueError("Interruption inference requires a semantics-v20 run")
|
| 224 |
+
if mode in {"emergency-stop", "safe-hold"}:
|
| 225 |
+
variants = {
|
| 226 |
+
"emergency-stop": int(
|
| 227 |
+
data.get("emergency_stop_session_variants", 0)
|
| 228 |
+
),
|
| 229 |
+
"safe-hold": int(data.get("safe_hold_session_variants", 0)),
|
| 230 |
+
}
|
| 231 |
+
if (
|
| 232 |
+
active_semantics < 21
|
| 233 |
+
or not bool(data.get("independent_task_pools", False))
|
| 234 |
+
or variants[mode] <= 0
|
| 235 |
+
):
|
| 236 |
+
raise ValueError(
|
| 237 |
+
f"{mode} inference requires an enabled semantics-v21 task pool"
|
| 238 |
+
)
|
| 239 |
+
control_turns = [
|
| 240 |
+
turn
|
| 241 |
+
for turn in turns
|
| 242 |
+
if str(turn.get("control_intent", "")).strip()
|
| 243 |
+
]
|
| 244 |
+
if len(control_turns) != 1 or str(
|
| 245 |
+
control_turns[0].get("control_intent")
|
| 246 |
+
).strip() != mode:
|
| 247 |
+
raise ValueError(
|
| 248 |
+
f"{mode} scenario needs exactly one matching control command"
|
| 249 |
+
)
|
| 250 |
+
if mode == "backchannel":
|
| 251 |
+
if (
|
| 252 |
+
active_semantics < 22
|
| 253 |
+
or not bool(data.get("independent_task_pools", False))
|
| 254 |
+
or int(data.get("backchannel_session_variants", 0)) <= 0
|
| 255 |
+
):
|
| 256 |
+
raise ValueError(
|
| 257 |
+
"Backchannel inference requires an enabled semantics-v22 pool"
|
| 258 |
+
)
|
| 259 |
+
if mode in {"hesitation", "pause"}:
|
| 260 |
+
variants = {
|
| 261 |
+
"hesitation": int(data.get("hesitation_session_variants", 0)),
|
| 262 |
+
"pause": int(data.get("pause_session_variants", 0)),
|
| 263 |
+
}
|
| 264 |
+
if (
|
| 265 |
+
active_semantics < 23
|
| 266 |
+
or not bool(data.get("independent_task_pools", False))
|
| 267 |
+
or variants[mode] <= 0
|
| 268 |
+
):
|
| 269 |
+
raise ValueError(
|
| 270 |
+
f"{mode} inference requires an enabled semantics-v23 task pool"
|
| 271 |
+
)
|
| 272 |
+
event = dict(payload.get("disfluency_event", {}))
|
| 273 |
+
if str(event.get("subtype", "")).strip().lower() != subtype:
|
| 274 |
+
raise ValueError(
|
| 275 |
+
f"{mode} scenario has no matching disfluency_event: {path}"
|
| 276 |
+
)
|
| 277 |
+
gap_steps = int(event.get("gap_steps", 0))
|
| 278 |
+
if not 1 <= gap_steps <= 7:
|
| 279 |
+
raise ValueError(
|
| 280 |
+
f"{mode} gap must use 1-7 Qwen steps, got {gap_steps}"
|
| 281 |
+
)
|
| 282 |
+
event_turn = int(event.get("turn_index", -1))
|
| 283 |
+
if not 0 <= event_turn < len(turns):
|
| 284 |
+
raise ValueError(f"Invalid disfluency event turn: {event_turn}")
|
| 285 |
+
if str(turns[event_turn].get("disfluency_subtype", "")) != subtype:
|
| 286 |
+
raise ValueError(
|
| 287 |
+
"Scenario event turn does not carry the declared disfluency subtype"
|
| 288 |
+
)
|
| 289 |
+
if active_semantics >= 24 and subtype == "self-repair":
|
| 290 |
+
repair_style = str(event.get("repair_style", "")).strip().lower()
|
| 291 |
+
if repair_style not in {"explicit-cue", "implicit-restart"}:
|
| 292 |
+
raise ValueError(
|
| 293 |
+
"Semantics-v24 self-repair scenario must declare "
|
| 294 |
+
f"repair_style, got {repair_style!r}"
|
| 295 |
+
)
|
| 296 |
+
marker_id = str(event.get("marker_id", "")).strip()
|
| 297 |
+
if (repair_style == "explicit-cue") != bool(marker_id):
|
| 298 |
+
raise ValueError(
|
| 299 |
+
"Explicit self-repair requires a marker and implicit "
|
| 300 |
+
"restart must not carry one"
|
| 301 |
+
)
|
| 302 |
+
|
| 303 |
+
if split != "custom":
|
| 304 |
+
split_seed = int(data.get("split_seed", -1))
|
| 305 |
+
val_ratio = float(data.get("val_ratio", -1.0))
|
| 306 |
+
namespace = str(data.get("split_source_namespace", "")).strip().strip("/")
|
| 307 |
+
expected_namespace = str(
|
| 308 |
+
payload.get("split_source_namespace", "")
|
| 309 |
+
).strip().strip("/")
|
| 310 |
+
if split_seed != int(payload.get("split_seed", -2)):
|
| 311 |
+
raise ValueError("Scenario/run split_seed mismatch")
|
| 312 |
+
if abs(val_ratio - float(payload.get("val_ratio", -2.0))) > 1e-12:
|
| 313 |
+
raise ValueError("Scenario/run val_ratio mismatch")
|
| 314 |
+
if not namespace or namespace != expected_namespace:
|
| 315 |
+
raise ValueError("Scenario/run split namespace mismatch")
|
| 316 |
+
for turn in turns:
|
| 317 |
+
filename = str(turn.get("source_motion_filename", "")).strip()
|
| 318 |
+
# Stop/hold utterances are control events, not new primitive
|
| 319 |
+
# motions. Their TTS path is audited above and deliberately has
|
| 320 |
+
# no train/val motion identity of its own.
|
| 321 |
+
if not filename and (
|
| 322 |
+
str(turn.get("control_intent", "")).strip()
|
| 323 |
+
or str(turn.get("backchannel_intent", "")).strip()
|
| 324 |
+
):
|
| 325 |
+
continue
|
| 326 |
+
if not filename:
|
| 327 |
+
raise ValueError("Split-tagged scenario turn has no motion filename")
|
| 328 |
+
observed = (
|
| 329 |
+
"val"
|
| 330 |
+
if _source_is_val(
|
| 331 |
+
_motion_source_id(namespace, filename),
|
| 332 |
+
split_seed=split_seed,
|
| 333 |
+
val_ratio=val_ratio,
|
| 334 |
+
)
|
| 335 |
+
else "train"
|
| 336 |
+
)
|
| 337 |
+
if observed != split:
|
| 338 |
+
raise ValueError(
|
| 339 |
+
f"Scenario source split mismatch: {filename} is {observed}, "
|
| 340 |
+
f"scenario claims {split}"
|
| 341 |
+
)
|
| 342 |
+
|
| 343 |
+
for turn in turns:
|
| 344 |
+
audio_path = Path(str(turn["audio_path"])).expanduser().resolve()
|
| 345 |
+
if not audio_path.is_file():
|
| 346 |
+
raise FileNotFoundError(f"Missing scenario audio: {audio_path}")
|
| 347 |
+
return InferenceCase(
|
| 348 |
+
name=str(payload["name"]),
|
| 349 |
+
interaction_mode=mode,
|
| 350 |
+
data_split=split,
|
| 351 |
+
scenario_path=path.resolve(),
|
| 352 |
+
scenario=payload,
|
| 353 |
+
interaction_subtype=subtype,
|
| 354 |
+
)
|
| 355 |
+
|
| 356 |
+
|
| 357 |
+
def load_cases(
|
| 358 |
+
suite_path: Path,
|
| 359 |
+
*,
|
| 360 |
+
run_config: dict[str, Any],
|
| 361 |
+
single_scenario: Path | None = None,
|
| 362 |
+
) -> list[InferenceCase]:
|
| 363 |
+
if single_scenario is not None:
|
| 364 |
+
return [validate_scenario(single_scenario, run_config=run_config)]
|
| 365 |
+
suite = _load_json(suite_path)
|
| 366 |
+
if suite.get("format") not in {
|
| 367 |
+
"hfd_multiturn_auto_infer_suite_v2",
|
| 368 |
+
"hfd_multiturn_auto_infer_suite_v3",
|
| 369 |
+
}:
|
| 370 |
+
raise ValueError(f"Unsupported inference suite: {suite_path}")
|
| 371 |
+
data = dict(run_config.get("data", {}))
|
| 372 |
+
enabled_variants = {
|
| 373 |
+
"turn-taking": int(data.get("turn_taking_session_variants", 0)),
|
| 374 |
+
"interruption": int(
|
| 375 |
+
data.get("command_interruption_session_variants", 0)
|
| 376 |
+
),
|
| 377 |
+
"emergency-stop": int(
|
| 378 |
+
data.get("emergency_stop_session_variants", 0)
|
| 379 |
+
),
|
| 380 |
+
"safe-hold": int(data.get("safe_hold_session_variants", 0)),
|
| 381 |
+
"backchannel": int(data.get("backchannel_session_variants", 0)),
|
| 382 |
+
"hesitation": int(data.get("hesitation_session_variants", 0)),
|
| 383 |
+
"pause": int(data.get("pause_session_variants", 0)),
|
| 384 |
+
}
|
| 385 |
+
enabled_modes = {
|
| 386 |
+
mode for mode, variants in enabled_variants.items() if variants > 0
|
| 387 |
+
}
|
| 388 |
+
cases: list[InferenceCase] = []
|
| 389 |
+
for item in suite.get("cases", []):
|
| 390 |
+
item_mode = str(item["interaction_mode"])
|
| 391 |
+
if item_mode not in enabled_modes:
|
| 392 |
+
continue
|
| 393 |
+
scenario_path = Path(str(item["scenario_file"]))
|
| 394 |
+
if not scenario_path.is_absolute():
|
| 395 |
+
scenario_path = (suite_path.parent / scenario_path).resolve()
|
| 396 |
+
cases.append(
|
| 397 |
+
validate_scenario(
|
| 398 |
+
scenario_path,
|
| 399 |
+
run_config=run_config,
|
| 400 |
+
expected_mode=item_mode,
|
| 401 |
+
expected_split=str(item["data_split"]),
|
| 402 |
+
expected_subtype=str(item.get("interaction_subtype", "")),
|
| 403 |
+
)
|
| 404 |
+
)
|
| 405 |
+
identities = {
|
| 406 |
+
(case.interaction_mode, case.interaction_subtype, case.data_split)
|
| 407 |
+
for case in cases
|
| 408 |
+
}
|
| 409 |
+
expected = set()
|
| 410 |
+
for mode in enabled_modes:
|
| 411 |
+
subtypes = (
|
| 412 |
+
("filled-pause", "self-repair")
|
| 413 |
+
if mode == "hesitation"
|
| 414 |
+
else ("silent-pause",)
|
| 415 |
+
if mode == "pause"
|
| 416 |
+
else ("",)
|
| 417 |
+
)
|
| 418 |
+
expected.update(
|
| 419 |
+
(mode, subtype, split)
|
| 420 |
+
for subtype in subtypes
|
| 421 |
+
for split in ("train", "val")
|
| 422 |
+
)
|
| 423 |
+
if identities != expected:
|
| 424 |
+
raise ValueError(
|
| 425 |
+
"Suite must contain train/val cases for every enabled task: "
|
| 426 |
+
f"expected={expected} observed={identities}"
|
| 427 |
+
)
|
| 428 |
+
return cases
|
| 429 |
+
|
| 430 |
+
|
| 431 |
+
def checkpoint_ready(path: Path) -> bool:
|
| 432 |
+
required = (
|
| 433 |
+
path / "model.safetensors",
|
| 434 |
+
path / "hfd_fused_delta_manifest.json",
|
| 435 |
+
path / "full_duplex_source_boundary.json",
|
| 436 |
+
)
|
| 437 |
+
if any(not item.is_file() or item.stat().st_size <= 0 for item in required):
|
| 438 |
+
return False
|
| 439 |
+
try:
|
| 440 |
+
marker = _load_json(required[-1])
|
| 441 |
+
except Exception:
|
| 442 |
+
return False
|
| 443 |
+
return bool(marker.get("source_group_complete", False))
|
| 444 |
+
|
| 445 |
+
|
| 446 |
+
def ready_checkpoint_steps(run_dir: Path) -> list[int]:
|
| 447 |
+
checkpoints = run_dir / "checkpoints"
|
| 448 |
+
found: list[int] = []
|
| 449 |
+
for path in checkpoints.glob("ckpt_[0-9]*"):
|
| 450 |
+
match = re.fullmatch(r"ckpt_([0-9]+)", path.name)
|
| 451 |
+
if match and path.is_dir() and checkpoint_ready(path):
|
| 452 |
+
found.append(int(match.group(1)))
|
| 453 |
+
return sorted(found)
|
| 454 |
+
|
| 455 |
+
|
| 456 |
+
def visible_gpus() -> list[str]:
|
| 457 |
+
configured = os.environ.get("HFD_INFER_GPUS", "").strip()
|
| 458 |
+
if not configured:
|
| 459 |
+
configured = os.environ.get("CUDA_VISIBLE_DEVICES", "").strip()
|
| 460 |
+
if configured:
|
| 461 |
+
values = [value.strip() for value in configured.split(",") if value.strip()]
|
| 462 |
+
else:
|
| 463 |
+
result = subprocess.run(
|
| 464 |
+
["nvidia-smi", "--query-gpu=index", "--format=csv,noheader"],
|
| 465 |
+
check=False,
|
| 466 |
+
text=True,
|
| 467 |
+
capture_output=True,
|
| 468 |
+
)
|
| 469 |
+
values = [line.strip() for line in result.stdout.splitlines() if line.strip()]
|
| 470 |
+
if not values:
|
| 471 |
+
raise RuntimeError("No inference GPU is visible")
|
| 472 |
+
if len(values) != len(set(values)):
|
| 473 |
+
raise ValueError(f"Duplicate GPU identifiers: {values}")
|
| 474 |
+
return values
|
| 475 |
+
|
| 476 |
+
|
| 477 |
+
def _safe_component(value: str) -> str:
|
| 478 |
+
return re.sub(r"[^A-Za-z0-9_.-]+", "_", value)
|
| 479 |
+
|
| 480 |
+
|
| 481 |
+
def case_result_dir(run_dir: Path, case: InferenceCase, step: int) -> Path:
|
| 482 |
+
turns = list(case.scenario["turns"])
|
| 483 |
+
# ``infer_render_auto.sh`` historically sanitizes ``echo "$stem"`` with
|
| 484 |
+
# ``tr -cs``. The terminating newline therefore becomes one trailing
|
| 485 |
+
# underscore. Reproduce that established on-disk name here rather than
|
| 486 |
+
# silently renaming or re-rendering every existing inference result.
|
| 487 |
+
audio_stem = _safe_component(Path(str(turns[0]["audio_path"])).stem + "\n")
|
| 488 |
+
sample_name = (
|
| 489 |
+
f"00_{_safe_component(case.name)}_{_safe_component(audio_stem)}_"
|
| 490 |
+
f"session{len(turns)}turn"
|
| 491 |
+
)
|
| 492 |
+
output_group = Path(case.interaction_mode)
|
| 493 |
+
if case.interaction_mode == "hesitation":
|
| 494 |
+
output_group /= (
|
| 495 |
+
"filled" if case.interaction_subtype == "filled-pause" else "self-repair"
|
| 496 |
+
)
|
| 497 |
+
return (
|
| 498 |
+
run_dir
|
| 499 |
+
/ output_group
|
| 500 |
+
/ f"ckpt_{step}"
|
| 501 |
+
/ case.data_split
|
| 502 |
+
/ sample_name
|
| 503 |
+
)
|
| 504 |
+
|
| 505 |
+
|
| 506 |
+
def case_settled(run_dir: Path, case: InferenceCase, step: int) -> bool:
|
| 507 |
+
result_dir = case_result_dir(run_dir, case, step)
|
| 508 |
+
return (result_dir / ".complete").is_file() or (result_dir / ".failed").is_file()
|
| 509 |
+
|
| 510 |
+
|
| 511 |
+
def case_failed(run_dir: Path, case: InferenceCase, step: int) -> bool:
|
| 512 |
+
return (case_result_dir(run_dir, case, step) / ".failed").is_file()
|
| 513 |
+
|
| 514 |
+
|
| 515 |
+
def case_environment(
|
| 516 |
+
run_dir: Path,
|
| 517 |
+
case: InferenceCase,
|
| 518 |
+
*,
|
| 519 |
+
step: int,
|
| 520 |
+
gpu: str,
|
| 521 |
+
) -> dict[str, str]:
|
| 522 |
+
payload = case.scenario
|
| 523 |
+
turns = list(payload["turns"])
|
| 524 |
+
audio_paths = [str(Path(str(turn["audio_path"])).expanduser().resolve()) for turn in turns]
|
| 525 |
+
environment = os.environ.copy()
|
| 526 |
+
output_group = Path(case.interaction_mode)
|
| 527 |
+
if case.interaction_mode == "hesitation":
|
| 528 |
+
output_group /= (
|
| 529 |
+
"filled" if case.interaction_subtype == "filled-pause" else "self-repair"
|
| 530 |
+
)
|
| 531 |
+
environment.update(
|
| 532 |
+
{
|
| 533 |
+
"CUDA_VISIBLE_DEVICES": gpu,
|
| 534 |
+
"HFD_INFER_GPUS": gpu,
|
| 535 |
+
"HFD_INFER_DEVICE": "cuda:0",
|
| 536 |
+
"HFD_INFER_ONCE": "1",
|
| 537 |
+
"HFD_INFER_EXACT_CHECKPOINT_STEP": str(step),
|
| 538 |
+
"HFD_INFER_MIN_CHECKPOINT_STEP": str(step),
|
| 539 |
+
"HFD_INFER_OUTPUT_ROOT": str(run_dir / output_group),
|
| 540 |
+
"HFD_INFER_SAMPLE_PARENT": case.data_split,
|
| 541 |
+
"HFD_INFER_SAMPLE_TAG": case.name,
|
| 542 |
+
"HFD_INFER_SCENARIO_FILE": str(case.scenario_path),
|
| 543 |
+
"HFD_INFER_AUDIO_PATHS": audio_paths[0],
|
| 544 |
+
"HFD_INFER_ADDITIONAL_AUDIO_PATHS": ",".join(audio_paths[1:]),
|
| 545 |
+
"HFD_INFER_INITIAL_IDLE_STEPS": str(payload["initial_idle_steps"]),
|
| 546 |
+
"HFD_INFER_MOTION_FRAMES": str(payload["motion_frames_per_turn"]),
|
| 547 |
+
"HFD_INFER_INTER_TURN_BACKGROUND_STEPS_BY_TURN": ",".join(
|
| 548 |
+
str(int(value))
|
| 549 |
+
for value in payload["inter_turn_background_steps_by_turn"]
|
| 550 |
+
),
|
| 551 |
+
"HFD_INFER_TERMINAL_BACKGROUND_STEPS": str(
|
| 552 |
+
payload["terminal_background_steps"]
|
| 553 |
+
),
|
| 554 |
+
}
|
| 555 |
+
)
|
| 556 |
+
init_motion = str(payload.get("init_motion_dir", "")).strip()
|
| 557 |
+
if init_motion:
|
| 558 |
+
environment["HFD_INFER_INIT_MOTION_DIRS"] = init_motion
|
| 559 |
+
return environment
|
| 560 |
+
|
| 561 |
+
|
| 562 |
+
def run_dynamic_queue(
|
| 563 |
+
run_dir: Path,
|
| 564 |
+
cases: list[InferenceCase],
|
| 565 |
+
*,
|
| 566 |
+
step: int,
|
| 567 |
+
gpus: list[str],
|
| 568 |
+
) -> None:
|
| 569 |
+
pending = deque(case for case in cases if not case_settled(run_dir, case, step))
|
| 570 |
+
available = deque(gpus)
|
| 571 |
+
running: dict[subprocess.Popen[bytes], tuple[InferenceCase, str]] = {}
|
| 572 |
+
failures: list[str] = []
|
| 573 |
+
base_script = PROJECT_ROOT / "infer_render_auto.sh"
|
| 574 |
+
while pending or running:
|
| 575 |
+
while pending and available:
|
| 576 |
+
case = pending.popleft()
|
| 577 |
+
gpu = available.popleft()
|
| 578 |
+
print(
|
| 579 |
+
f"[suite] start ckpt_{step} {case.interaction_mode}/{case.data_split} "
|
| 580 |
+
f"on GPU {gpu}",
|
| 581 |
+
flush=True,
|
| 582 |
+
)
|
| 583 |
+
process = subprocess.Popen(
|
| 584 |
+
["bash", str(base_script), str(run_dir)],
|
| 585 |
+
cwd=PROJECT_ROOT,
|
| 586 |
+
env=case_environment(run_dir, case, step=step, gpu=gpu),
|
| 587 |
+
)
|
| 588 |
+
running[process] = (case, gpu)
|
| 589 |
+
finished = [process for process in running if process.poll() is not None]
|
| 590 |
+
if not finished:
|
| 591 |
+
time.sleep(0.25)
|
| 592 |
+
continue
|
| 593 |
+
for process in finished:
|
| 594 |
+
case, gpu = running.pop(process)
|
| 595 |
+
available.append(gpu)
|
| 596 |
+
if process.returncode != 0:
|
| 597 |
+
failures.append(
|
| 598 |
+
f"{case.interaction_mode}/{case.data_split} "
|
| 599 |
+
f"exited {process.returncode}"
|
| 600 |
+
)
|
| 601 |
+
elif case_failed(run_dir, case, step):
|
| 602 |
+
failures.append(
|
| 603 |
+
f"{case.interaction_mode}/{case.data_split} wrote .failed"
|
| 604 |
+
)
|
| 605 |
+
elif not case_settled(run_dir, case, step):
|
| 606 |
+
failures.append(
|
| 607 |
+
f"{case.interaction_mode}/{case.data_split} wrote no marker"
|
| 608 |
+
)
|
| 609 |
+
else:
|
| 610 |
+
print(
|
| 611 |
+
f"[suite] done ckpt_{step} "
|
| 612 |
+
f"{case.interaction_mode}/{case.data_split} on GPU {gpu}",
|
| 613 |
+
flush=True,
|
| 614 |
+
)
|
| 615 |
+
if failures:
|
| 616 |
+
raise RuntimeError(
|
| 617 |
+
f"ckpt_{step} inference suite failed: " + "; ".join(failures)
|
| 618 |
+
)
|
| 619 |
+
|
| 620 |
+
|
| 621 |
+
def main() -> None:
|
| 622 |
+
if len(sys.argv) != 2:
|
| 623 |
+
raise SystemExit(f"usage: {Path(sys.argv[0]).name} RUN_DIR")
|
| 624 |
+
run_dir = Path(sys.argv[1]).expanduser().resolve()
|
| 625 |
+
run_config_path = run_dir / "run_config.json"
|
| 626 |
+
if not run_config_path.is_file():
|
| 627 |
+
raise SystemExit(f"Invalid HFD run directory: {run_dir}")
|
| 628 |
+
run_config = _load_json(run_config_path)
|
| 629 |
+
configured_suite = os.environ.get("HFD_MULTITURN_SUITE_FILE", "").strip()
|
| 630 |
+
active_semantics = int(
|
| 631 |
+
dict(run_config.get("train", {})).get(
|
| 632 |
+
"full_duplex_semantics_version",
|
| 633 |
+
0,
|
| 634 |
+
)
|
| 635 |
+
)
|
| 636 |
+
suite_path = Path(
|
| 637 |
+
configured_suite
|
| 638 |
+
or (DEFAULT_SUITE_V24 if active_semantics >= 24 else DEFAULT_SUITE_V23)
|
| 639 |
+
)
|
| 640 |
+
suite_path = suite_path.expanduser().resolve()
|
| 641 |
+
single_raw = os.environ.get("HFD_MULTITURN_SCENARIO_FILE", "").strip()
|
| 642 |
+
single_scenario = Path(single_raw).expanduser().resolve() if single_raw else None
|
| 643 |
+
cases = load_cases(
|
| 644 |
+
suite_path,
|
| 645 |
+
run_config=run_config,
|
| 646 |
+
single_scenario=single_scenario,
|
| 647 |
+
)
|
| 648 |
+
gpus = visible_gpus()
|
| 649 |
+
check_interval = max(int(os.environ.get("HFD_INFER_CHECK_INTERVAL", "30")), 1)
|
| 650 |
+
infer_once = _truthy(os.environ.get("HFD_INFER_ONCE"))
|
| 651 |
+
exact_raw = os.environ.get("HFD_INFER_EXACT_CHECKPOINT_STEP", "").strip()
|
| 652 |
+
exact_step = int(exact_raw) if exact_raw else None
|
| 653 |
+
minimum_raw = os.environ.get("HFD_INFER_MIN_CHECKPOINT_STEP", "auto").strip()
|
| 654 |
+
initial_steps = ready_checkpoint_steps(run_dir)
|
| 655 |
+
if minimum_raw == "auto":
|
| 656 |
+
minimum_step = initial_steps[-1] if initial_steps else 0
|
| 657 |
+
else:
|
| 658 |
+
minimum_step = int(minimum_raw)
|
| 659 |
+
print(
|
| 660 |
+
f"AUTO INFER SUITE: cases={len(cases)} GPUs={','.join(gpus)} "
|
| 661 |
+
f"min_step={minimum_step} exact_step={exact_step or 'none'}",
|
| 662 |
+
flush=True,
|
| 663 |
+
)
|
| 664 |
+
|
| 665 |
+
while True:
|
| 666 |
+
ready = ready_checkpoint_steps(run_dir)
|
| 667 |
+
if exact_step is not None:
|
| 668 |
+
candidates = [exact_step] if exact_step in ready else []
|
| 669 |
+
else:
|
| 670 |
+
candidates = [step for step in ready if step >= minimum_step]
|
| 671 |
+
pending_steps = [
|
| 672 |
+
step
|
| 673 |
+
for step in candidates
|
| 674 |
+
if any(not case_settled(run_dir, case, step) for case in cases)
|
| 675 |
+
]
|
| 676 |
+
if pending_steps:
|
| 677 |
+
step = max(pending_steps)
|
| 678 |
+
run_dynamic_queue(run_dir, cases, step=step, gpus=gpus)
|
| 679 |
+
if infer_once or exact_step is not None:
|
| 680 |
+
break
|
| 681 |
+
else:
|
| 682 |
+
failed = [
|
| 683 |
+
f"{case.interaction_mode}/{case.data_split}"
|
| 684 |
+
for step in candidates
|
| 685 |
+
for case in cases
|
| 686 |
+
if case_failed(run_dir, case, step)
|
| 687 |
+
]
|
| 688 |
+
if failed:
|
| 689 |
+
raise RuntimeError(
|
| 690 |
+
"Existing inference failure marker(s) require inspection or "
|
| 691 |
+
f"cleanup before retry: {', '.join(failed)}"
|
| 692 |
+
)
|
| 693 |
+
print(f"[{time.strftime('%F %T')}] no pending checkpoint", flush=True)
|
| 694 |
+
if infer_once or exact_step is not None:
|
| 695 |
+
break
|
| 696 |
+
time.sleep(check_interval)
|
| 697 |
+
|
| 698 |
+
|
| 699 |
+
if __name__ == "__main__":
|
| 700 |
+
main()
|
scripts/infer/seed_audio_to_motion.py
ADDED
|
@@ -0,0 +1,530 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#!/usr/bin/env python3
|
| 2 |
+
from __future__ import annotations
|
| 3 |
+
|
| 4 |
+
import argparse
|
| 5 |
+
import json
|
| 6 |
+
import math
|
| 7 |
+
import random
|
| 8 |
+
import re
|
| 9 |
+
import shutil
|
| 10 |
+
import subprocess
|
| 11 |
+
from pathlib import Path
|
| 12 |
+
from typing import List
|
| 13 |
+
|
| 14 |
+
import numpy as np
|
| 15 |
+
import torch
|
| 16 |
+
|
| 17 |
+
from psi.config.config import LaunchConfig
|
| 18 |
+
from psi.data.seed.motion_repr import quat_inv_wxyz, quat_mul_wxyz, recover_seed_abs_from_textop_action
|
| 19 |
+
from psi.data.seed.progress import build_progress_conditions
|
| 20 |
+
from psi.models.psi0 import Psi0Model
|
| 21 |
+
from psi.utils import parse_args_to_tyro_config, seed_everything
|
| 22 |
+
|
| 23 |
+
from seed_text_to_motion import (
|
| 24 |
+
TARGET_FPS,
|
| 25 |
+
build_model_states_from_abs,
|
| 26 |
+
finite_difference,
|
| 27 |
+
load_pose_from_motion_dir,
|
| 28 |
+
load_source_frame_count,
|
| 29 |
+
normalize_quaternion_rows,
|
| 30 |
+
quaternion_angular_velocity,
|
| 31 |
+
recover_seed_abs_from_textop_action_with_joint_delta,
|
| 32 |
+
resolve_ckpt_step,
|
| 33 |
+
sanitize_motion_name,
|
| 34 |
+
should_decode_textop_joint_from_delta,
|
| 35 |
+
write_csv,
|
| 36 |
+
write_metadata,
|
| 37 |
+
)
|
| 38 |
+
|
| 39 |
+
|
| 40 |
+
def cfg_get(obj, key: str, default=None):
|
| 41 |
+
if isinstance(obj, dict):
|
| 42 |
+
return obj.get(key, default)
|
| 43 |
+
return getattr(obj, key, default)
|
| 44 |
+
|
| 45 |
+
|
| 46 |
+
def audio_data_cfg(launch_config: LaunchConfig, key: str = "audio"):
|
| 47 |
+
return cfg_get(launch_config.data, key, launch_config.data)
|
| 48 |
+
|
| 49 |
+
|
| 50 |
+
def load_audio_array(video_path: Path, sampling_rate: int) -> np.ndarray:
|
| 51 |
+
cmd = [
|
| 52 |
+
"ffmpeg",
|
| 53 |
+
"-v",
|
| 54 |
+
"error",
|
| 55 |
+
"-i",
|
| 56 |
+
str(video_path),
|
| 57 |
+
"-ac",
|
| 58 |
+
"1",
|
| 59 |
+
"-ar",
|
| 60 |
+
str(sampling_rate),
|
| 61 |
+
"-f",
|
| 62 |
+
"f32le",
|
| 63 |
+
"-",
|
| 64 |
+
]
|
| 65 |
+
proc = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, check=False)
|
| 66 |
+
if proc.returncode != 0:
|
| 67 |
+
msg = proc.stderr.decode("utf-8", errors="ignore")
|
| 68 |
+
raise RuntimeError(f"ffmpeg failed to decode audio from {video_path}: {msg}")
|
| 69 |
+
audio = np.frombuffer(proc.stdout, dtype=np.float32).copy()
|
| 70 |
+
if audio.size == 0:
|
| 71 |
+
raise RuntimeError(f"No audio samples decoded from {video_path}")
|
| 72 |
+
return audio
|
| 73 |
+
|
| 74 |
+
|
| 75 |
+
def audio_segment(audio: np.ndarray, sampling_rate: int, start_sec: float, duration_sec: float) -> np.ndarray:
|
| 76 |
+
num_samples = max(1, int(round(duration_sec * sampling_rate)))
|
| 77 |
+
start_sample = int(round(start_sec * sampling_rate))
|
| 78 |
+
end_sample = start_sample + num_samples
|
| 79 |
+
|
| 80 |
+
segment = np.zeros((num_samples,), dtype=np.float32)
|
| 81 |
+
src_start = max(0, start_sample)
|
| 82 |
+
src_end = min(audio.shape[0], end_sample)
|
| 83 |
+
if src_end > src_start:
|
| 84 |
+
dst_start = src_start - start_sample
|
| 85 |
+
segment[dst_start : dst_start + (src_end - src_start)] = audio[src_start:src_end]
|
| 86 |
+
return segment
|
| 87 |
+
|
| 88 |
+
|
| 89 |
+
def read_start_hold_seconds(motion_dir: Path, align_start_hold: bool) -> float:
|
| 90 |
+
if not align_start_hold:
|
| 91 |
+
return 0.0
|
| 92 |
+
info_path = motion_dir / "info.txt"
|
| 93 |
+
if not info_path.exists():
|
| 94 |
+
return 0.0
|
| 95 |
+
text = info_path.read_text(encoding="utf-8", errors="ignore")
|
| 96 |
+
match = re.search(r"start_hold_seconds:\s*([0-9.]+)", text)
|
| 97 |
+
return float(match.group(1)) if match else 0.0
|
| 98 |
+
|
| 99 |
+
|
| 100 |
+
def resolve_motion_dir(root_dir: Path, motion_dir_arg: str) -> Path:
|
| 101 |
+
motion_dir = Path(motion_dir_arg).expanduser()
|
| 102 |
+
if motion_dir.is_absolute():
|
| 103 |
+
return motion_dir
|
| 104 |
+
return root_dir / motion_dir
|
| 105 |
+
|
| 106 |
+
|
| 107 |
+
def resolve_video_path(video_dir: Path, motion_dir: Path, video_path_arg: str) -> Path:
|
| 108 |
+
if video_path_arg:
|
| 109 |
+
video_path = Path(video_path_arg).expanduser()
|
| 110 |
+
if video_path.is_absolute():
|
| 111 |
+
return video_path
|
| 112 |
+
return video_dir / video_path
|
| 113 |
+
return video_dir / f"{motion_dir.name}.mp4"
|
| 114 |
+
|
| 115 |
+
|
| 116 |
+
def resolve_audio_path(audio_path: Path, extensions: str = ".wav,.flac,.mp3,.m4a,.aac,.ogg") -> Path:
|
| 117 |
+
if audio_path.is_file():
|
| 118 |
+
return audio_path
|
| 119 |
+
if not audio_path.is_dir():
|
| 120 |
+
return audio_path
|
| 121 |
+
audio_exts = tuple(
|
| 122 |
+
ext if ext.startswith(".") else f".{ext}"
|
| 123 |
+
for ext in [part.strip().lower() for part in str(extensions).split(",")]
|
| 124 |
+
if ext
|
| 125 |
+
)
|
| 126 |
+
candidates = sorted(p for p in audio_path.iterdir() if p.is_file() and p.suffix.lower() in audio_exts)
|
| 127 |
+
if not candidates:
|
| 128 |
+
raise FileNotFoundError(f"No audio files found under {audio_path}")
|
| 129 |
+
return random.choice(candidates)
|
| 130 |
+
|
| 131 |
+
|
| 132 |
+
def normalize_action_for_rtc(maxmin, action: np.ndarray) -> np.ndarray:
|
| 133 |
+
action_min = np.asarray(maxmin.action_min, dtype=np.float32)
|
| 134 |
+
action_max = np.asarray(maxmin.action_max, dtype=np.float32)
|
| 135 |
+
ill_mask = np.abs(action_max - action_min) < 1e-4 * (np.abs(action_max) + np.abs(action_min) + 1e-8)
|
| 136 |
+
action_max = action_max.copy()
|
| 137 |
+
action_max[ill_mask] = 1.0
|
| 138 |
+
action_norm = np.where(ill_mask, action, (action - action_min) / (action_max - action_min) * 2 - 1)
|
| 139 |
+
if bool(getattr(maxmin, "use_norm_mask", False)):
|
| 140 |
+
action_norm_masks = np.asarray(maxmin.action_norm_masks, dtype=bool)
|
| 141 |
+
action_norm = np.where(action_norm_masks, action_norm, action)
|
| 142 |
+
return np.clip(action_norm, -1.0, 1.0).astype(np.float32)
|
| 143 |
+
|
| 144 |
+
|
| 145 |
+
def rollout_audio_motion(
|
| 146 |
+
*,
|
| 147 |
+
model: Psi0Model,
|
| 148 |
+
maxmin,
|
| 149 |
+
odim: int,
|
| 150 |
+
device: str,
|
| 151 |
+
instruction: str,
|
| 152 |
+
audio: np.ndarray,
|
| 153 |
+
audio_sampling_rate: int,
|
| 154 |
+
audio_window_seconds: float,
|
| 155 |
+
audio_left_context_seconds: float,
|
| 156 |
+
full_audio: bool,
|
| 157 |
+
start_hold_seconds: float,
|
| 158 |
+
normalize_state: bool,
|
| 159 |
+
textop_actions: bool,
|
| 160 |
+
decode_textop_joint_from_delta: bool,
|
| 161 |
+
num_past_frames: int,
|
| 162 |
+
use_future_progress_embedding: bool,
|
| 163 |
+
enable_rtc: bool,
|
| 164 |
+
action_exec_horizon: int,
|
| 165 |
+
rtc_max_delay: int,
|
| 166 |
+
num_inference_steps: int,
|
| 167 |
+
target_frames: int,
|
| 168 |
+
init_abs_state: np.ndarray,
|
| 169 |
+
) -> tuple[np.ndarray, np.ndarray, int]:
|
| 170 |
+
horizon = int(model.action_horizon)
|
| 171 |
+
exec_horizon = int(action_exec_horizon) if enable_rtc else horizon
|
| 172 |
+
if exec_horizon <= 0 or exec_horizon > horizon:
|
| 173 |
+
raise ValueError(f"Invalid action_exec_horizon={exec_horizon}, action_horizon={horizon}")
|
| 174 |
+
rtc_delay = horizon - exec_horizon
|
| 175 |
+
if enable_rtc and rtc_delay > 0 and rtc_delay >= int(rtc_max_delay):
|
| 176 |
+
raise ValueError(
|
| 177 |
+
f"RTC requires action_horizon - action_exec_horizon < max_delay, "
|
| 178 |
+
f"got {horizon} - {exec_horizon} = {rtc_delay}, max_delay={rtc_max_delay}"
|
| 179 |
+
)
|
| 180 |
+
if not textop_actions:
|
| 181 |
+
raise ValueError("This audio inference path is for the 39D TextOp action checkpoint.")
|
| 182 |
+
if target_frames <= 0:
|
| 183 |
+
target_frames = max(exec_horizon, int(round(audio.shape[0] / float(audio_sampling_rate) * TARGET_FPS)))
|
| 184 |
+
|
| 185 |
+
num_chunks = int(math.ceil(float(target_frames) / float(exec_horizon)))
|
| 186 |
+
current_abs = np.asarray(init_abs_state, dtype=np.float32).copy()
|
| 187 |
+
state_horizon = int(num_past_frames) + 1
|
| 188 |
+
state_history: List[np.ndarray] = [current_abs.copy() for _ in range(state_horizon)]
|
| 189 |
+
all_abs: List[np.ndarray] = []
|
| 190 |
+
all_delta: List[np.ndarray] = []
|
| 191 |
+
prev_actions_norm: np.ndarray | None = None
|
| 192 |
+
remaining_frames = int(target_frames)
|
| 193 |
+
|
| 194 |
+
for _ in range(num_chunks):
|
| 195 |
+
frame_cursor = target_frames - remaining_frames
|
| 196 |
+
states = build_model_states_from_abs(
|
| 197 |
+
state_history[-state_horizon:],
|
| 198 |
+
textop_actions=textop_actions,
|
| 199 |
+
odim=odim,
|
| 200 |
+
)
|
| 201 |
+
if normalize_state:
|
| 202 |
+
states = maxmin.normalize_state_func(states)
|
| 203 |
+
states_t = torch.from_numpy(states).unsqueeze(0).to(device)
|
| 204 |
+
future_progress_t = None
|
| 205 |
+
if use_future_progress_embedding:
|
| 206 |
+
_, future_progress = build_progress_conditions(
|
| 207 |
+
frame_id=frame_cursor,
|
| 208 |
+
total_frames=target_frames,
|
| 209 |
+
action_horizon=horizon,
|
| 210 |
+
)
|
| 211 |
+
future_progress_t = torch.from_numpy(future_progress[None]).to(device)
|
| 212 |
+
|
| 213 |
+
# <modify_full_audio> 和训练一致:full_audio=True时每个chunk都看到整段音频embedding,不再给time/progress。
|
| 214 |
+
if full_audio:
|
| 215 |
+
audio_clip = audio.astype(np.float32)
|
| 216 |
+
else:
|
| 217 |
+
audio_start_sec = (
|
| 218 |
+
frame_cursor / float(TARGET_FPS)
|
| 219 |
+
- float(start_hold_seconds)
|
| 220 |
+
- float(audio_left_context_seconds)
|
| 221 |
+
)
|
| 222 |
+
audio_clip = audio_segment(audio, audio_sampling_rate, audio_start_sec, audio_window_seconds)
|
| 223 |
+
|
| 224 |
+
with torch.inference_mode():
|
| 225 |
+
if enable_rtc and prev_actions_norm is not None and rtc_delay > 0:
|
| 226 |
+
prev_rtc = np.concatenate(
|
| 227 |
+
[
|
| 228 |
+
prev_actions_norm[exec_horizon:, :],
|
| 229 |
+
np.zeros((exec_horizon, prev_actions_norm.shape[1]), dtype=np.float32),
|
| 230 |
+
],
|
| 231 |
+
axis=0,
|
| 232 |
+
)
|
| 233 |
+
prev_actions_t = torch.from_numpy(prev_rtc[None]).to(device)
|
| 234 |
+
pred_norm = model.predict_action_with_training_rtc_flow(
|
| 235 |
+
observations=None,
|
| 236 |
+
states=states_t,
|
| 237 |
+
instructions=[instruction],
|
| 238 |
+
num_inference_steps=num_inference_steps,
|
| 239 |
+
traj2ds=None,
|
| 240 |
+
prev_actions=prev_actions_t,
|
| 241 |
+
inference_delay=rtc_delay,
|
| 242 |
+
max_delay=int(rtc_max_delay),
|
| 243 |
+
audios=[audio_clip],
|
| 244 |
+
future_progress=future_progress_t,
|
| 245 |
+
)[0].float().cpu().numpy()
|
| 246 |
+
else:
|
| 247 |
+
pred_norm = model.predict_action(
|
| 248 |
+
observations=None,
|
| 249 |
+
states=states_t,
|
| 250 |
+
instructions=[instruction],
|
| 251 |
+
num_inference_steps=num_inference_steps,
|
| 252 |
+
traj2ds=None,
|
| 253 |
+
audios=[audio_clip],
|
| 254 |
+
future_progress=future_progress_t,
|
| 255 |
+
)[0].float().cpu().numpy()
|
| 256 |
+
|
| 257 |
+
pred_norm_clipped = np.clip(np.asarray(pred_norm, dtype=np.float32), -1.0, 1.0)
|
| 258 |
+
chunk_pred = np.asarray(maxmin.denormalize(pred_norm_clipped), dtype=np.float32)
|
| 259 |
+
|
| 260 |
+
if decode_textop_joint_from_delta:
|
| 261 |
+
chunk_abs = recover_seed_abs_from_textop_action_with_joint_delta(chunk_pred, current_abs)
|
| 262 |
+
else:
|
| 263 |
+
chunk_abs = recover_seed_abs_from_textop_action(chunk_pred, current_abs)
|
| 264 |
+
|
| 265 |
+
prev_abs = np.concatenate([current_abs[None, :], chunk_abs[:-1]], axis=0)
|
| 266 |
+
joint_delta = chunk_abs[:, :29] - prev_abs[:, :29]
|
| 267 |
+
root_pos_delta = chunk_abs[:, 29:32] - prev_abs[:, 29:32]
|
| 268 |
+
root_quat_delta = quat_mul_wxyz(quat_inv_wxyz(prev_abs[:, 32:36]), chunk_abs[:, 32:36])
|
| 269 |
+
chunk_delta = np.concatenate([joint_delta, root_pos_delta, root_quat_delta], axis=1)
|
| 270 |
+
|
| 271 |
+
chunk_abs = normalize_quaternion_rows(chunk_abs.astype(np.float32))
|
| 272 |
+
if decode_textop_joint_from_delta:
|
| 273 |
+
chunk_pred_for_rtc = chunk_pred.copy()
|
| 274 |
+
chunk_pred_for_rtc[:, :29] = chunk_abs[:, :29]
|
| 275 |
+
prev_actions_norm = normalize_action_for_rtc(maxmin, chunk_pred_for_rtc)
|
| 276 |
+
else:
|
| 277 |
+
prev_actions_norm = pred_norm_clipped.copy()
|
| 278 |
+
|
| 279 |
+
take = min(exec_horizon, remaining_frames)
|
| 280 |
+
if take <= 0:
|
| 281 |
+
break
|
| 282 |
+
executed_abs = chunk_abs[:take]
|
| 283 |
+
executed_delta = chunk_delta[:take].astype(np.float32)
|
| 284 |
+
current_abs = executed_abs[-1].copy()
|
| 285 |
+
for frame_abs in executed_abs:
|
| 286 |
+
state_history.append(frame_abs.astype(np.float32).copy())
|
| 287 |
+
state_history = state_history[-state_horizon:]
|
| 288 |
+
all_abs.append(executed_abs)
|
| 289 |
+
all_delta.append(executed_delta)
|
| 290 |
+
remaining_frames -= take
|
| 291 |
+
|
| 292 |
+
pred_abs = np.concatenate(all_abs, axis=0)[:target_frames]
|
| 293 |
+
pred_delta = np.concatenate(all_delta, axis=0)[:target_frames]
|
| 294 |
+
pred_abs = normalize_quaternion_rows(pred_abs.astype(np.float32))
|
| 295 |
+
return pred_abs, pred_delta, num_chunks
|
| 296 |
+
|
| 297 |
+
|
| 298 |
+
def write_audio_info(
|
| 299 |
+
path: Path,
|
| 300 |
+
*,
|
| 301 |
+
audio_config_key: str,
|
| 302 |
+
source_motion_dir: Path,
|
| 303 |
+
video_path: Path,
|
| 304 |
+
initial_pose_source: str,
|
| 305 |
+
frames: int,
|
| 306 |
+
textop_actions: bool,
|
| 307 |
+
num_past_frames: int,
|
| 308 |
+
enable_rtc: bool,
|
| 309 |
+
action_exec_horizon: int,
|
| 310 |
+
num_inference_steps: int,
|
| 311 |
+
audio_sampling_rate: int,
|
| 312 |
+
audio_window_seconds: float,
|
| 313 |
+
full_audio: bool,
|
| 314 |
+
instruction: str,
|
| 315 |
+
) -> None:
|
| 316 |
+
lines = [
|
| 317 |
+
"Motion Information: seed_audio_to_motion_inference",
|
| 318 |
+
"===================================================",
|
| 319 |
+
"",
|
| 320 |
+
f"audio_config_key: {audio_config_key}",
|
| 321 |
+
f"source_motion_dir: {source_motion_dir}",
|
| 322 |
+
f"video_path: {video_path}",
|
| 323 |
+
f"instruction: {instruction}",
|
| 324 |
+
f"initial_pose_source: {initial_pose_source}",
|
| 325 |
+
f"target_fps: {TARGET_FPS}",
|
| 326 |
+
f"num_inference_steps: {num_inference_steps}",
|
| 327 |
+
f"trained_with_textop_actions: {textop_actions}",
|
| 328 |
+
f"num_past_frames: {num_past_frames}",
|
| 329 |
+
f"rtc_enabled: {enable_rtc}",
|
| 330 |
+
f"action_exec_horizon: {action_exec_horizon}",
|
| 331 |
+
f"audio_sampling_rate: {audio_sampling_rate}",
|
| 332 |
+
f"full_audio: {full_audio}",
|
| 333 |
+
f"audio_window_seconds: {audio_window_seconds}",
|
| 334 |
+
"output_root_quaternion: wxyz",
|
| 335 |
+
"output_joint_angles: 29D",
|
| 336 |
+
"output_root_position: xyz",
|
| 337 |
+
"",
|
| 338 |
+
f"frames: {frames}",
|
| 339 |
+
"",
|
| 340 |
+
]
|
| 341 |
+
path.write_text("\n".join(lines), encoding="utf-8")
|
| 342 |
+
|
| 343 |
+
|
| 344 |
+
def parse_args() -> argparse.Namespace:
|
| 345 |
+
parser = argparse.ArgumentParser(description="Seed audio->motion offline inference exporter")
|
| 346 |
+
parser.add_argument("--run-dir", type=str, required=True)
|
| 347 |
+
parser.add_argument("--ckpt-step", type=str, default="latest", help="integer step, 'latest', or 'best'")
|
| 348 |
+
parser.add_argument("--output-dir", type=str, required=True)
|
| 349 |
+
parser.add_argument("--motion-dir", type=str, required=True)
|
| 350 |
+
parser.add_argument("--video-path", type=str, default="")
|
| 351 |
+
parser.add_argument(
|
| 352 |
+
"--audio-config-key",
|
| 353 |
+
type=str,
|
| 354 |
+
default="audio",
|
| 355 |
+
choices=("audio", "audio_command"),
|
| 356 |
+
help="Which launch_config.data audio section to use",
|
| 357 |
+
)
|
| 358 |
+
parser.add_argument("--prompt", type=str, default="", help="Optional text instruction paired with the audio")
|
| 359 |
+
parser.add_argument("--num-output-frames", type=int, default=0, help="0 means source motion length")
|
| 360 |
+
parser.add_argument("--num-inference-steps", type=int, default=10)
|
| 361 |
+
parser.add_argument("--device", type=str, default="cuda:0")
|
| 362 |
+
parser.add_argument("--seed", type=int, default=42)
|
| 363 |
+
parser.add_argument("--export-delta", action="store_true", default=False)
|
| 364 |
+
parser.add_argument("--disable-rtc", action="store_true", default=False)
|
| 365 |
+
parser.add_argument("--no-align-start-hold", dest="align_start_hold", action="store_false")
|
| 366 |
+
parser.set_defaults(align_start_hold=True)
|
| 367 |
+
return parser.parse_args()
|
| 368 |
+
|
| 369 |
+
|
| 370 |
+
def main() -> None:
|
| 371 |
+
args = parse_args()
|
| 372 |
+
|
| 373 |
+
run_dir = Path(args.run_dir)
|
| 374 |
+
output_dir = Path(args.output_dir)
|
| 375 |
+
output_dir.mkdir(parents=True, exist_ok=True)
|
| 376 |
+
|
| 377 |
+
cfg_template: LaunchConfig = parse_args_to_tyro_config(run_dir / "argv.txt") # type: ignore
|
| 378 |
+
launch_config = cfg_template.model_validate_json((run_dir / "run_config.json").read_text())
|
| 379 |
+
seed_everything(launch_config.seed or args.seed)
|
| 380 |
+
|
| 381 |
+
ckpt_step = resolve_ckpt_step(run_dir, args.ckpt_step)
|
| 382 |
+
model = Psi0Model.from_pretrained(run_dir, ckpt_step, launch_config, device=args.device)
|
| 383 |
+
model.text_only = False
|
| 384 |
+
model.audio_only = True
|
| 385 |
+
model.video_only = False
|
| 386 |
+
model.to(args.device)
|
| 387 |
+
model.eval()
|
| 388 |
+
|
| 389 |
+
audio_cfg = audio_data_cfg(launch_config, args.audio_config_key)
|
| 390 |
+
data_root = Path(cfg_get(audio_cfg, "root_dir"))
|
| 391 |
+
video_root = Path(cfg_get(audio_cfg, "video_dir", ""))
|
| 392 |
+
motion_dir = resolve_motion_dir(data_root, args.motion_dir)
|
| 393 |
+
video_path = resolve_audio_path(
|
| 394 |
+
resolve_video_path(video_root, motion_dir, args.video_path),
|
| 395 |
+
str(cfg_get(audio_cfg, "audio_file_extensions", ".wav,.flac,.mp3,.m4a,.aac,.ogg")),
|
| 396 |
+
)
|
| 397 |
+
if not motion_dir.exists():
|
| 398 |
+
raise FileNotFoundError(f"motion_dir does not exist: {motion_dir}")
|
| 399 |
+
if not video_path.exists():
|
| 400 |
+
raise FileNotFoundError(f"video_path does not exist: {video_path}")
|
| 401 |
+
|
| 402 |
+
maxmin = launch_config.data.transform.field
|
| 403 |
+
odim = int(launch_config.model.odim)
|
| 404 |
+
textop_actions = bool(getattr(launch_config.data.transform.repack, "textop_actions", False))
|
| 405 |
+
decode_textop_joint_from_delta = textop_actions and should_decode_textop_joint_from_delta(launch_config)
|
| 406 |
+
num_past_frames = int(getattr(launch_config.data.transform.repack, "num_past_frames", 0))
|
| 407 |
+
use_future_progress_embedding = bool(getattr(launch_config.model, "use_future_progress_embedding", False))
|
| 408 |
+
enable_rtc = bool(getattr(launch_config.model, "rtc", False)) and (not args.disable_rtc)
|
| 409 |
+
action_exec_horizon = int(getattr(launch_config.model, "action_exec_horizon", int(model.action_horizon)))
|
| 410 |
+
rtc_max_delay = int(getattr(launch_config.model, "max_delay", 0))
|
| 411 |
+
audio_sampling_rate = int(cfg_get(audio_cfg, "audio_sampling_rate", 16000))
|
| 412 |
+
audio_window_seconds = float(cfg_get(audio_cfg, "audio_window_seconds", 1.0))
|
| 413 |
+
audio_left_context_seconds = float(cfg_get(audio_cfg, "audio_left_context_seconds", 0.0))
|
| 414 |
+
full_audio = bool(cfg_get(audio_cfg, "full_audio", False))
|
| 415 |
+
|
| 416 |
+
audio = load_audio_array(video_path, audio_sampling_rate)
|
| 417 |
+
instruction = args.prompt.strip()
|
| 418 |
+
x0 = load_pose_from_motion_dir(motion_dir)
|
| 419 |
+
source_frames = load_source_frame_count(motion_dir)
|
| 420 |
+
target_frames = int(args.num_output_frames) if args.num_output_frames > 0 else int(source_frames or 0)
|
| 421 |
+
start_hold_seconds = read_start_hold_seconds(motion_dir, bool(args.align_start_hold))
|
| 422 |
+
|
| 423 |
+
pred_abs, pred_delta, rollout_chunks = rollout_audio_motion(
|
| 424 |
+
model=model,
|
| 425 |
+
maxmin=maxmin,
|
| 426 |
+
odim=odim,
|
| 427 |
+
device=args.device,
|
| 428 |
+
instruction=instruction,
|
| 429 |
+
audio=audio,
|
| 430 |
+
audio_sampling_rate=audio_sampling_rate,
|
| 431 |
+
audio_window_seconds=audio_window_seconds,
|
| 432 |
+
audio_left_context_seconds=audio_left_context_seconds,
|
| 433 |
+
full_audio=full_audio,
|
| 434 |
+
start_hold_seconds=start_hold_seconds,
|
| 435 |
+
normalize_state=bool(getattr(maxmin, "normalize_state", False)),
|
| 436 |
+
textop_actions=textop_actions,
|
| 437 |
+
decode_textop_joint_from_delta=decode_textop_joint_from_delta,
|
| 438 |
+
num_past_frames=num_past_frames,
|
| 439 |
+
use_future_progress_embedding=use_future_progress_embedding,
|
| 440 |
+
enable_rtc=enable_rtc,
|
| 441 |
+
action_exec_horizon=action_exec_horizon,
|
| 442 |
+
rtc_max_delay=rtc_max_delay,
|
| 443 |
+
num_inference_steps=int(args.num_inference_steps),
|
| 444 |
+
target_frames=target_frames,
|
| 445 |
+
init_abs_state=x0,
|
| 446 |
+
)
|
| 447 |
+
|
| 448 |
+
motion_name = sanitize_motion_name(motion_dir.name)
|
| 449 |
+
sample_dir = output_dir / motion_name
|
| 450 |
+
sample_dir.mkdir(parents=True, exist_ok=True)
|
| 451 |
+
|
| 452 |
+
joint_pos = pred_abs[:, :29]
|
| 453 |
+
body_pos = pred_abs[:, 29:32]
|
| 454 |
+
body_quat = pred_abs[:, 32:36]
|
| 455 |
+
|
| 456 |
+
dt = 1.0 / float(TARGET_FPS)
|
| 457 |
+
joint_vel = finite_difference(joint_pos, dt)
|
| 458 |
+
body_lin_vel = finite_difference(body_pos, dt)
|
| 459 |
+
body_ang_vel = quaternion_angular_velocity(body_quat, dt)
|
| 460 |
+
|
| 461 |
+
write_csv(sample_dir / "joint_pos.csv", joint_pos, [f"joint_{i}" for i in range(29)])
|
| 462 |
+
write_csv(sample_dir / "joint_vel.csv", joint_vel, [f"joint_vel_{i}" for i in range(29)])
|
| 463 |
+
write_csv(sample_dir / "body_pos.csv", body_pos, ["body_0_x", "body_0_y", "body_0_z"])
|
| 464 |
+
write_csv(sample_dir / "body_quat.csv", body_quat, ["body_0_w", "body_0_x", "body_0_y", "body_0_z"])
|
| 465 |
+
write_csv(sample_dir / "body_lin_vel.csv", body_lin_vel, ["body_0_vel_x", "body_0_vel_y", "body_0_vel_z"])
|
| 466 |
+
write_csv(sample_dir / "body_ang_vel.csv", body_ang_vel, ["body_0_angvel_x", "body_0_angvel_y", "body_0_angvel_z"])
|
| 467 |
+
np.save(sample_dir / "motion.npy", pred_abs)
|
| 468 |
+
|
| 469 |
+
if args.export_delta:
|
| 470 |
+
write_csv(sample_dir / "joint_pos_delta.csv", pred_delta[:, :29], [f"joint_{i}" for i in range(29)])
|
| 471 |
+
write_csv(sample_dir / "body_pos_delta.csv", pred_delta[:, 29:32], ["body_0_x", "body_0_y", "body_0_z"])
|
| 472 |
+
write_csv(sample_dir / "body_quat_delta.csv", pred_delta[:, 32:36], ["body_0_w", "body_0_x", "body_0_y", "body_0_z"])
|
| 473 |
+
np.save(sample_dir / "motion_delta.npy", pred_delta)
|
| 474 |
+
|
| 475 |
+
write_metadata(sample_dir / "metadata.txt", motion_name, int(pred_abs.shape[0]))
|
| 476 |
+
write_audio_info(
|
| 477 |
+
sample_dir / "info.txt",
|
| 478 |
+
audio_config_key=str(args.audio_config_key),
|
| 479 |
+
source_motion_dir=motion_dir,
|
| 480 |
+
video_path=video_path,
|
| 481 |
+
initial_pose_source=str(motion_dir),
|
| 482 |
+
frames=int(pred_abs.shape[0]),
|
| 483 |
+
textop_actions=textop_actions,
|
| 484 |
+
num_past_frames=int(num_past_frames),
|
| 485 |
+
enable_rtc=enable_rtc,
|
| 486 |
+
action_exec_horizon=int(action_exec_horizon),
|
| 487 |
+
num_inference_steps=int(args.num_inference_steps),
|
| 488 |
+
audio_sampling_rate=int(audio_sampling_rate),
|
| 489 |
+
audio_window_seconds=float(audio_window_seconds),
|
| 490 |
+
full_audio=bool(full_audio),
|
| 491 |
+
instruction=instruction,
|
| 492 |
+
)
|
| 493 |
+
|
| 494 |
+
for src_name, dst_name in (("info.txt", "info_source.txt"), ("metadata.txt", "metadata_source.txt")):
|
| 495 |
+
src_path = motion_dir / src_name
|
| 496 |
+
if src_path.exists():
|
| 497 |
+
shutil.copy2(src_path, sample_dir / dst_name)
|
| 498 |
+
|
| 499 |
+
sample_meta = {
|
| 500 |
+
"run_dir": str(run_dir),
|
| 501 |
+
"ckpt_step": ckpt_step,
|
| 502 |
+
"audio_config_key": str(args.audio_config_key),
|
| 503 |
+
"instruction": instruction,
|
| 504 |
+
"source_motion_dir": str(motion_dir),
|
| 505 |
+
"video_path": str(video_path),
|
| 506 |
+
"output_dir": str(sample_dir),
|
| 507 |
+
"num_output_frames": int(pred_abs.shape[0]),
|
| 508 |
+
"rollout_chunks": int(rollout_chunks),
|
| 509 |
+
"textop_actions": textop_actions,
|
| 510 |
+
"decode_textop_joint_from_delta": decode_textop_joint_from_delta,
|
| 511 |
+
"num_past_frames": int(num_past_frames),
|
| 512 |
+
"enable_rtc": enable_rtc,
|
| 513 |
+
"action_exec_horizon": int(action_exec_horizon),
|
| 514 |
+
"audio_sampling_rate": int(audio_sampling_rate),
|
| 515 |
+
"full_audio": bool(full_audio),
|
| 516 |
+
"audio_window_seconds": float(audio_window_seconds),
|
| 517 |
+
"audio_left_context_seconds": float(audio_left_context_seconds),
|
| 518 |
+
"start_hold_seconds": float(start_hold_seconds),
|
| 519 |
+
}
|
| 520 |
+
(sample_dir / "infer_meta.json").write_text(json.dumps(sample_meta, ensure_ascii=False, indent=2), encoding="utf-8")
|
| 521 |
+
(output_dir / "manifest.json").write_text(
|
| 522 |
+
json.dumps({"samples": [sample_meta]}, ensure_ascii=False, indent=2),
|
| 523 |
+
encoding="utf-8",
|
| 524 |
+
)
|
| 525 |
+
print(f"[OK] audio sample -> {sample_dir}")
|
| 526 |
+
print(f"[DONE] wrote manifest: {output_dir / 'manifest.json'}")
|
| 527 |
+
|
| 528 |
+
|
| 529 |
+
if __name__ == "__main__":
|
| 530 |
+
main()
|
scripts/infer/seed_motion_teacher_reconstruct.py
ADDED
|
@@ -0,0 +1,504 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#!/usr/bin/env python3
|
| 2 |
+
from __future__ import annotations
|
| 3 |
+
|
| 4 |
+
import argparse
|
| 5 |
+
import json
|
| 6 |
+
import math
|
| 7 |
+
import shutil
|
| 8 |
+
from pathlib import Path
|
| 9 |
+
from typing import List
|
| 10 |
+
|
| 11 |
+
import numpy as np
|
| 12 |
+
import pandas as pd
|
| 13 |
+
import torch
|
| 14 |
+
import torch.nn as nn
|
| 15 |
+
from safetensors import safe_open
|
| 16 |
+
|
| 17 |
+
from psi.config.config import LaunchConfig
|
| 18 |
+
from psi.data.seed.motion_repr import (
|
| 19 |
+
build_seed_textop_action,
|
| 20 |
+
quat_inv_wxyz,
|
| 21 |
+
quat_mul_wxyz,
|
| 22 |
+
recover_seed_abs_from_textop_action,
|
| 23 |
+
)
|
| 24 |
+
from psi.data.seed.progress import build_progress_conditions
|
| 25 |
+
from psi.models.motion_distill import MotionDistillPsi0Model
|
| 26 |
+
from psi.utils import parse_args_to_tyro_config, seed_everything
|
| 27 |
+
|
| 28 |
+
from seed_text_to_motion import (
|
| 29 |
+
TARGET_FPS,
|
| 30 |
+
build_model_states_from_abs,
|
| 31 |
+
finite_difference,
|
| 32 |
+
load_source_frame_count,
|
| 33 |
+
normalize_quaternion_rows,
|
| 34 |
+
quaternion_angular_velocity,
|
| 35 |
+
recover_seed_abs_from_textop_action_with_joint_delta,
|
| 36 |
+
resolve_ckpt_step,
|
| 37 |
+
sanitize_motion_name,
|
| 38 |
+
should_decode_textop_joint_from_delta,
|
| 39 |
+
write_csv,
|
| 40 |
+
write_metadata,
|
| 41 |
+
)
|
| 42 |
+
|
| 43 |
+
|
| 44 |
+
def resolve_checkpoint_dir(run_dir: Path, ckpt_step: int | str) -> Path:
|
| 45 |
+
if str(ckpt_step) in {"best", "best_ckpt"}:
|
| 46 |
+
ckpt_dir = run_dir / "checkpoints" / "best_ckpt"
|
| 47 |
+
else:
|
| 48 |
+
ckpt_dir = run_dir / "checkpoints" / f"ckpt_{ckpt_step}"
|
| 49 |
+
if not ckpt_dir.exists():
|
| 50 |
+
root_ckpt_dir = run_dir / f"ckpt_{ckpt_step}"
|
| 51 |
+
if root_ckpt_dir.exists():
|
| 52 |
+
ckpt_dir = root_ckpt_dir
|
| 53 |
+
if not ckpt_dir.exists():
|
| 54 |
+
raise FileNotFoundError(f"Checkpoint dir does not exist: {ckpt_dir}")
|
| 55 |
+
return ckpt_dir
|
| 56 |
+
|
| 57 |
+
|
| 58 |
+
def load_selected_checkpoint_state(ckpt_dir: Path, prefixes: tuple[str, ...]) -> dict[str, torch.Tensor]:
|
| 59 |
+
model_path = ckpt_dir / "model.safetensors"
|
| 60 |
+
index_path = ckpt_dir / "model.safetensors.index.json"
|
| 61 |
+
shard_paths: list[Path]
|
| 62 |
+
|
| 63 |
+
if index_path.exists():
|
| 64 |
+
weight_map = json.loads(index_path.read_text()).get("weight_map", {})
|
| 65 |
+
shard_paths = sorted(
|
| 66 |
+
{ckpt_dir / shard for key, shard in weight_map.items() if key.startswith(prefixes)}
|
| 67 |
+
)
|
| 68 |
+
elif model_path.exists():
|
| 69 |
+
shard_paths = [model_path]
|
| 70 |
+
else:
|
| 71 |
+
raise FileNotFoundError(f"No safetensors checkpoint found under {ckpt_dir}")
|
| 72 |
+
|
| 73 |
+
state_dict: dict[str, torch.Tensor] = {}
|
| 74 |
+
for shard_path in shard_paths:
|
| 75 |
+
with safe_open(str(shard_path), framework="pt", device="cpu") as f:
|
| 76 |
+
for key in f.keys():
|
| 77 |
+
if key.startswith(prefixes):
|
| 78 |
+
state_dict[key] = f.get_tensor(key)
|
| 79 |
+
return state_dict
|
| 80 |
+
|
| 81 |
+
|
| 82 |
+
def build_noise_scheduler(launch_config: LaunchConfig):
|
| 83 |
+
if launch_config.model.noise_scheduler == "ddpm":
|
| 84 |
+
from diffusers.schedulers.scheduling_ddim import DDIMScheduler
|
| 85 |
+
|
| 86 |
+
return DDIMScheduler(
|
| 87 |
+
num_train_timesteps=launch_config.model.train_diffusion_steps,
|
| 88 |
+
beta_start=0.0001,
|
| 89 |
+
beta_end=0.02,
|
| 90 |
+
beta_schedule="squaredcos_cap_v2",
|
| 91 |
+
clip_sample=True,
|
| 92 |
+
set_alpha_to_one=True,
|
| 93 |
+
steps_offset=0,
|
| 94 |
+
prediction_type="epsilon",
|
| 95 |
+
)
|
| 96 |
+
if launch_config.model.noise_scheduler == "flow":
|
| 97 |
+
from diffusers.schedulers.scheduling_flow_match_euler_discrete import FlowMatchEulerDiscreteScheduler
|
| 98 |
+
|
| 99 |
+
return FlowMatchEulerDiscreteScheduler(
|
| 100 |
+
num_train_timesteps=launch_config.model.train_diffusion_steps,
|
| 101 |
+
)
|
| 102 |
+
raise ValueError(f"Unsupported noise scheduler: {launch_config.model.noise_scheduler}")
|
| 103 |
+
|
| 104 |
+
|
| 105 |
+
def load_teacher_reconstruct_model(
|
| 106 |
+
run_dir: Path,
|
| 107 |
+
ckpt_step: int | str,
|
| 108 |
+
launch_config: LaunchConfig,
|
| 109 |
+
device: str,
|
| 110 |
+
) -> MotionDistillPsi0Model:
|
| 111 |
+
ckpt_dir = resolve_checkpoint_dir(run_dir, ckpt_step)
|
| 112 |
+
state_dict = load_selected_checkpoint_state(
|
| 113 |
+
ckpt_dir,
|
| 114 |
+
prefixes=("action_header.", "motion_encoder."),
|
| 115 |
+
)
|
| 116 |
+
model = MotionDistillPsi0Model(
|
| 117 |
+
model_cfg=launch_config.model,
|
| 118 |
+
vlm_model=nn.Module(),
|
| 119 |
+
)
|
| 120 |
+
action_state = {
|
| 121 |
+
key[len("action_header.") :]: value
|
| 122 |
+
for key, value in state_dict.items()
|
| 123 |
+
if key.startswith("action_header.")
|
| 124 |
+
}
|
| 125 |
+
motion_state = {
|
| 126 |
+
key[len("motion_encoder.") :]: value
|
| 127 |
+
for key, value in state_dict.items()
|
| 128 |
+
if key.startswith("motion_encoder.")
|
| 129 |
+
}
|
| 130 |
+
if not action_state:
|
| 131 |
+
raise ValueError(f"No action_header weights found in {ckpt_dir}")
|
| 132 |
+
if not motion_state:
|
| 133 |
+
raise ValueError(f"No motion_encoder weights found in {ckpt_dir}")
|
| 134 |
+
model.action_header.load_state_dict(action_state, strict=True)
|
| 135 |
+
model.motion_encoder.load_state_dict(motion_state, strict=True)
|
| 136 |
+
model.noise_scheduler = build_noise_scheduler(launch_config)
|
| 137 |
+
model.action_horizon = int(launch_config.model.action_chunk_size)
|
| 138 |
+
model.action_dim = int(launch_config.model.action_dim)
|
| 139 |
+
model.device = device
|
| 140 |
+
return model.to(device)
|
| 141 |
+
|
| 142 |
+
|
| 143 |
+
def load_motion_abs(motion_dir: Path) -> np.ndarray:
|
| 144 |
+
joint = pd.read_csv(motion_dir / "joint_pos.csv").to_numpy(dtype=np.float32)
|
| 145 |
+
body_pos = pd.read_csv(motion_dir / "body_pos.csv").to_numpy(dtype=np.float32)
|
| 146 |
+
body_quat = pd.read_csv(motion_dir / "body_quat.csv").to_numpy(dtype=np.float32)
|
| 147 |
+
if not (len(joint) == len(body_pos) == len(body_quat)):
|
| 148 |
+
raise ValueError(f"Motion csv length mismatch under {motion_dir}")
|
| 149 |
+
return np.concatenate([joint, body_pos, body_quat], axis=1).astype(np.float32)
|
| 150 |
+
|
| 151 |
+
|
| 152 |
+
def build_abs_action_chunk(
|
| 153 |
+
motion_abs: np.ndarray,
|
| 154 |
+
frame_id: int,
|
| 155 |
+
horizon: int,
|
| 156 |
+
sample_stride: int = 1,
|
| 157 |
+
) -> np.ndarray:
|
| 158 |
+
total_frames = int(motion_abs.shape[0])
|
| 159 |
+
actions: List[np.ndarray] = []
|
| 160 |
+
for step in range(horizon):
|
| 161 |
+
raw_index = frame_id + (step + 1) * sample_stride
|
| 162 |
+
actions.append(motion_abs[min(raw_index, total_frames - 1)])
|
| 163 |
+
return np.stack(actions, axis=0).astype(np.float32)
|
| 164 |
+
|
| 165 |
+
|
| 166 |
+
def build_teacher_action_features(
|
| 167 |
+
actions_abs: np.ndarray,
|
| 168 |
+
current_abs: np.ndarray,
|
| 169 |
+
*,
|
| 170 |
+
textop_actions: bool,
|
| 171 |
+
action_dim: int,
|
| 172 |
+
) -> np.ndarray:
|
| 173 |
+
if textop_actions:
|
| 174 |
+
actions = build_seed_textop_action(actions_abs[:, :36], current_abs[:36])
|
| 175 |
+
else:
|
| 176 |
+
actions = actions_abs[:, :action_dim]
|
| 177 |
+
|
| 178 |
+
if actions.shape[-1] < action_dim:
|
| 179 |
+
pad = np.zeros((actions.shape[0], action_dim - actions.shape[-1]), dtype=np.float32)
|
| 180 |
+
actions = np.concatenate([actions, pad], axis=-1)
|
| 181 |
+
if actions.shape[-1] != action_dim:
|
| 182 |
+
raise ValueError(f"Teacher action dim mismatch: built {actions.shape[-1]}, expected {action_dim}")
|
| 183 |
+
return actions.astype(np.float32)
|
| 184 |
+
|
| 185 |
+
|
| 186 |
+
def normalize_action_only(maxmin, actions: np.ndarray) -> np.ndarray:
|
| 187 |
+
action_min = np.asarray(maxmin.action_min, dtype=np.float32)
|
| 188 |
+
action_max = np.asarray(maxmin.action_max, dtype=np.float32)
|
| 189 |
+
ill_mask = np.abs(action_max - action_min) < 1e-4 * (
|
| 190 |
+
np.abs(action_max) + np.abs(action_min) + 1e-8
|
| 191 |
+
)
|
| 192 |
+
denom = action_max - action_min
|
| 193 |
+
denom[ill_mask] = 1.0
|
| 194 |
+
actions_norm = np.where(
|
| 195 |
+
ill_mask,
|
| 196 |
+
actions,
|
| 197 |
+
(actions - action_min) / denom * 2 - 1,
|
| 198 |
+
)
|
| 199 |
+
if bool(getattr(maxmin, "use_norm_mask", False)):
|
| 200 |
+
action_norm_masks = np.asarray(maxmin.action_norm_masks, dtype=bool)
|
| 201 |
+
actions_norm = np.where(action_norm_masks, actions_norm, actions)
|
| 202 |
+
return np.clip(actions_norm, -1.0, 1.0).astype(np.float32)
|
| 203 |
+
|
| 204 |
+
|
| 205 |
+
def decode_action_chunk(
|
| 206 |
+
chunk_pred: np.ndarray,
|
| 207 |
+
current_abs: np.ndarray,
|
| 208 |
+
*,
|
| 209 |
+
textop_actions: bool,
|
| 210 |
+
decode_textop_joint_from_delta: bool,
|
| 211 |
+
) -> tuple[np.ndarray, np.ndarray]:
|
| 212 |
+
if textop_actions:
|
| 213 |
+
if decode_textop_joint_from_delta:
|
| 214 |
+
chunk_abs = recover_seed_abs_from_textop_action_with_joint_delta(chunk_pred, current_abs)
|
| 215 |
+
else:
|
| 216 |
+
chunk_abs = recover_seed_abs_from_textop_action(chunk_pred, current_abs)
|
| 217 |
+
prev_abs = np.concatenate([current_abs[None, :], chunk_abs[:-1]], axis=0)
|
| 218 |
+
joint_delta = chunk_abs[:, :29] - prev_abs[:, :29]
|
| 219 |
+
root_pos_delta = chunk_abs[:, 29:32] - prev_abs[:, 29:32]
|
| 220 |
+
root_quat_delta = quat_mul_wxyz(quat_inv_wxyz(prev_abs[:, 32:36]), chunk_abs[:, 32:36])
|
| 221 |
+
chunk_delta = np.concatenate([joint_delta, root_pos_delta, root_quat_delta], axis=1)
|
| 222 |
+
else:
|
| 223 |
+
chunk_abs = chunk_pred[:, :36]
|
| 224 |
+
first_delta = chunk_abs[0] - current_abs
|
| 225 |
+
if chunk_abs.shape[0] > 1:
|
| 226 |
+
rest_delta = chunk_abs[1:] - chunk_abs[:-1]
|
| 227 |
+
chunk_delta = np.concatenate([first_delta[None, :], rest_delta], axis=0)
|
| 228 |
+
else:
|
| 229 |
+
chunk_delta = first_delta[None, :]
|
| 230 |
+
|
| 231 |
+
return normalize_quaternion_rows(chunk_abs.astype(np.float32)), chunk_delta.astype(np.float32)
|
| 232 |
+
|
| 233 |
+
|
| 234 |
+
def reconstruct_teacher_motion(
|
| 235 |
+
*,
|
| 236 |
+
model: MotionDistillPsi0Model,
|
| 237 |
+
maxmin,
|
| 238 |
+
motion_abs: np.ndarray,
|
| 239 |
+
odim: int,
|
| 240 |
+
action_dim: int,
|
| 241 |
+
device: str,
|
| 242 |
+
normalize_state: bool,
|
| 243 |
+
textop_actions: bool,
|
| 244 |
+
decode_textop_joint_from_delta: bool,
|
| 245 |
+
num_past_frames: int,
|
| 246 |
+
use_future_progress_embedding: bool,
|
| 247 |
+
num_inference_steps: int,
|
| 248 |
+
target_frames: int,
|
| 249 |
+
) -> tuple[np.ndarray, np.ndarray, dict[str, float]]:
|
| 250 |
+
horizon = int(model.action_horizon)
|
| 251 |
+
if target_frames <= 0:
|
| 252 |
+
target_frames = max(1, int(motion_abs.shape[0]) - 1)
|
| 253 |
+
|
| 254 |
+
num_chunks = int(math.ceil(float(target_frames) / float(horizon)))
|
| 255 |
+
all_abs: List[np.ndarray] = []
|
| 256 |
+
all_delta: List[np.ndarray] = []
|
| 257 |
+
remaining_frames = int(target_frames)
|
| 258 |
+
|
| 259 |
+
for chunk_id in range(num_chunks):
|
| 260 |
+
frame_id = chunk_id * horizon
|
| 261 |
+
current_abs = motion_abs[min(frame_id, motion_abs.shape[0] - 1)].astype(np.float32)
|
| 262 |
+
state_indices = [
|
| 263 |
+
max(0, frame_id - (int(num_past_frames) - t))
|
| 264 |
+
for t in range(int(num_past_frames) + 1)
|
| 265 |
+
]
|
| 266 |
+
states = build_model_states_from_abs(
|
| 267 |
+
[motion_abs[min(i, motion_abs.shape[0] - 1)] for i in state_indices],
|
| 268 |
+
textop_actions=textop_actions,
|
| 269 |
+
odim=odim,
|
| 270 |
+
)
|
| 271 |
+
if normalize_state:
|
| 272 |
+
states = maxmin.normalize_state_func(states)
|
| 273 |
+
|
| 274 |
+
actions_abs = build_abs_action_chunk(motion_abs, frame_id, horizon)
|
| 275 |
+
teacher_actions = build_teacher_action_features(
|
| 276 |
+
actions_abs,
|
| 277 |
+
current_abs,
|
| 278 |
+
textop_actions=textop_actions,
|
| 279 |
+
action_dim=action_dim,
|
| 280 |
+
)
|
| 281 |
+
teacher_actions_norm = normalize_action_only(maxmin, teacher_actions)
|
| 282 |
+
|
| 283 |
+
with torch.inference_mode():
|
| 284 |
+
states_t = torch.from_numpy(states).unsqueeze(0).to(device)
|
| 285 |
+
teacher_actions_t = torch.from_numpy(teacher_actions_norm).unsqueeze(0).to(device)
|
| 286 |
+
hidden_states = model.encode_motion_hidden(teacher_actions_t)
|
| 287 |
+
|
| 288 |
+
future_progress_t = None
|
| 289 |
+
if use_future_progress_embedding:
|
| 290 |
+
_, future_progress = build_progress_conditions(
|
| 291 |
+
frame_id=frame_id,
|
| 292 |
+
total_frames=int(motion_abs.shape[0]),
|
| 293 |
+
action_horizon=horizon,
|
| 294 |
+
)
|
| 295 |
+
future_progress_t = torch.from_numpy(future_progress[None]).to(device)
|
| 296 |
+
|
| 297 |
+
action_samples = torch.randn(1, horizon, action_dim, device=device)
|
| 298 |
+
model.noise_scheduler.set_timesteps(num_inference_steps)
|
| 299 |
+
for timestep in model.noise_scheduler.timesteps:
|
| 300 |
+
batched_timestep = timestep.expand(1).to(device)
|
| 301 |
+
model_output = model.action_from_hidden(
|
| 302 |
+
hidden_states=hidden_states,
|
| 303 |
+
attention_mask=None,
|
| 304 |
+
action_samples=action_samples,
|
| 305 |
+
states=states_t,
|
| 306 |
+
timestep=batched_timestep,
|
| 307 |
+
traj2ds=None,
|
| 308 |
+
future_progress=future_progress_t,
|
| 309 |
+
return_dict=True,
|
| 310 |
+
)
|
| 311 |
+
action_samples = model.noise_scheduler.step(
|
| 312 |
+
model_output=model_output.action,
|
| 313 |
+
timestep=timestep,
|
| 314 |
+
sample=action_samples,
|
| 315 |
+
).prev_sample
|
| 316 |
+
|
| 317 |
+
pred_norm = action_samples[0].float().cpu().numpy()
|
| 318 |
+
pred_norm_clipped = np.clip(np.asarray(pred_norm, dtype=np.float32), -1.0, 1.0)
|
| 319 |
+
chunk_pred = np.asarray(maxmin.denormalize(pred_norm_clipped), dtype=np.float32)
|
| 320 |
+
chunk_abs, chunk_delta = decode_action_chunk(
|
| 321 |
+
chunk_pred,
|
| 322 |
+
current_abs,
|
| 323 |
+
textop_actions=textop_actions,
|
| 324 |
+
decode_textop_joint_from_delta=decode_textop_joint_from_delta,
|
| 325 |
+
)
|
| 326 |
+
|
| 327 |
+
take = min(horizon, remaining_frames)
|
| 328 |
+
all_abs.append(chunk_abs[:take])
|
| 329 |
+
all_delta.append(chunk_delta[:take])
|
| 330 |
+
remaining_frames -= take
|
| 331 |
+
|
| 332 |
+
pred_abs = np.concatenate(all_abs, axis=0)[:target_frames]
|
| 333 |
+
pred_delta = np.concatenate(all_delta, axis=0)[:target_frames]
|
| 334 |
+
pred_abs = normalize_quaternion_rows(pred_abs.astype(np.float32))
|
| 335 |
+
|
| 336 |
+
ref_abs = motion_abs[1 : 1 + pred_abs.shape[0], :36]
|
| 337 |
+
if ref_abs.shape[0] == pred_abs.shape[0]:
|
| 338 |
+
diff = pred_abs - ref_abs
|
| 339 |
+
metrics = {
|
| 340 |
+
"joint_mse": float(np.mean(diff[:, :29] ** 2)),
|
| 341 |
+
"root_pos_mse": float(np.mean(diff[:, 29:32] ** 2)),
|
| 342 |
+
"all_abs_mse": float(np.mean(diff ** 2)),
|
| 343 |
+
}
|
| 344 |
+
else:
|
| 345 |
+
metrics = {}
|
| 346 |
+
return pred_abs, pred_delta, metrics
|
| 347 |
+
|
| 348 |
+
|
| 349 |
+
def write_teacher_info(
|
| 350 |
+
path: Path,
|
| 351 |
+
*,
|
| 352 |
+
source_motion_dir: Path,
|
| 353 |
+
frames: int,
|
| 354 |
+
textop_actions: bool,
|
| 355 |
+
num_past_frames: int,
|
| 356 |
+
num_inference_steps: int,
|
| 357 |
+
metrics: dict[str, float],
|
| 358 |
+
) -> None:
|
| 359 |
+
lines = [
|
| 360 |
+
"Motion Information: motion_distill_teacher_reconstruct",
|
| 361 |
+
"======================================================",
|
| 362 |
+
"",
|
| 363 |
+
f"source_motion_dir: {source_motion_dir}",
|
| 364 |
+
f"target_fps: {TARGET_FPS}",
|
| 365 |
+
f"num_inference_steps: {num_inference_steps}",
|
| 366 |
+
f"trained_with_textop_actions: {textop_actions}",
|
| 367 |
+
f"num_past_frames: {num_past_frames}",
|
| 368 |
+
"condition: motion_encoder(ground_truth_actions)",
|
| 369 |
+
"output_root_quaternion: wxyz",
|
| 370 |
+
"output_joint_angles: 29D",
|
| 371 |
+
"output_root_position: xyz",
|
| 372 |
+
"",
|
| 373 |
+
f"frames: {frames}",
|
| 374 |
+
"",
|
| 375 |
+
]
|
| 376 |
+
for key, value in metrics.items():
|
| 377 |
+
lines.append(f"{key}: {value:.8f}")
|
| 378 |
+
path.write_text("\n".join(lines), encoding="utf-8")
|
| 379 |
+
|
| 380 |
+
|
| 381 |
+
def parse_args() -> argparse.Namespace:
|
| 382 |
+
parser = argparse.ArgumentParser(description="Motion-distill teacher motion reconstruction exporter")
|
| 383 |
+
parser.add_argument("--run-dir", type=str, required=True)
|
| 384 |
+
parser.add_argument("--ckpt-step", type=str, default="best", help="integer step, 'latest', or 'best'")
|
| 385 |
+
parser.add_argument("--output-dir", type=str, required=True)
|
| 386 |
+
parser.add_argument("--motion-dir", type=str, required=True)
|
| 387 |
+
parser.add_argument("--num-output-frames", type=int, default=0, help="0 means source motion length minus one")
|
| 388 |
+
parser.add_argument("--num-inference-steps", type=int, default=10)
|
| 389 |
+
parser.add_argument("--device", type=str, default="cuda:0")
|
| 390 |
+
parser.add_argument("--seed", type=int, default=42)
|
| 391 |
+
parser.add_argument("--export-delta", action="store_true", default=False)
|
| 392 |
+
return parser.parse_args()
|
| 393 |
+
|
| 394 |
+
|
| 395 |
+
def main() -> None:
|
| 396 |
+
args = parse_args()
|
| 397 |
+
run_dir = Path(args.run_dir)
|
| 398 |
+
output_dir = Path(args.output_dir)
|
| 399 |
+
motion_dir = Path(args.motion_dir)
|
| 400 |
+
output_dir.mkdir(parents=True, exist_ok=True)
|
| 401 |
+
|
| 402 |
+
cfg_template: LaunchConfig = parse_args_to_tyro_config(run_dir / "argv.txt") # type: ignore
|
| 403 |
+
launch_config = cfg_template.model_validate_json((run_dir / "run_config.json").read_text())
|
| 404 |
+
seed_everything(launch_config.seed or args.seed)
|
| 405 |
+
|
| 406 |
+
ckpt_step = resolve_ckpt_step(run_dir, args.ckpt_step)
|
| 407 |
+
model = load_teacher_reconstruct_model(run_dir, ckpt_step, launch_config, device=args.device)
|
| 408 |
+
model.eval()
|
| 409 |
+
|
| 410 |
+
motion_abs = load_motion_abs(motion_dir)
|
| 411 |
+
source_frames = load_source_frame_count(motion_dir) or int(motion_abs.shape[0])
|
| 412 |
+
target_frames = int(args.num_output_frames) if args.num_output_frames > 0 else max(1, source_frames - 1)
|
| 413 |
+
|
| 414 |
+
maxmin = launch_config.data.transform.field
|
| 415 |
+
odim = int(launch_config.model.odim)
|
| 416 |
+
action_dim = int(getattr(launch_config.model, "action_dim", getattr(model, "action_dim", 39)))
|
| 417 |
+
textop_actions = bool(getattr(launch_config.data.transform.repack, "textop_actions", False))
|
| 418 |
+
decode_textop_joint_from_delta = textop_actions and should_decode_textop_joint_from_delta(launch_config)
|
| 419 |
+
num_past_frames = int(getattr(launch_config.data.transform.repack, "num_past_frames", 0))
|
| 420 |
+
use_future_progress_embedding = bool(getattr(launch_config.model, "use_future_progress_embedding", False))
|
| 421 |
+
|
| 422 |
+
pred_abs, pred_delta, metrics = reconstruct_teacher_motion(
|
| 423 |
+
model=model,
|
| 424 |
+
maxmin=maxmin,
|
| 425 |
+
motion_abs=motion_abs,
|
| 426 |
+
odim=odim,
|
| 427 |
+
action_dim=action_dim,
|
| 428 |
+
device=args.device,
|
| 429 |
+
normalize_state=bool(getattr(maxmin, "normalize_state", False)),
|
| 430 |
+
textop_actions=textop_actions,
|
| 431 |
+
decode_textop_joint_from_delta=decode_textop_joint_from_delta,
|
| 432 |
+
num_past_frames=num_past_frames,
|
| 433 |
+
use_future_progress_embedding=use_future_progress_embedding,
|
| 434 |
+
num_inference_steps=int(args.num_inference_steps),
|
| 435 |
+
target_frames=target_frames,
|
| 436 |
+
)
|
| 437 |
+
|
| 438 |
+
motion_name = sanitize_motion_name(motion_dir.name)
|
| 439 |
+
sample_dir = output_dir / motion_name
|
| 440 |
+
sample_dir.mkdir(parents=True, exist_ok=True)
|
| 441 |
+
|
| 442 |
+
joint_pos = pred_abs[:, :29]
|
| 443 |
+
body_pos = pred_abs[:, 29:32]
|
| 444 |
+
body_quat = pred_abs[:, 32:36]
|
| 445 |
+
|
| 446 |
+
dt = 1.0 / float(TARGET_FPS)
|
| 447 |
+
joint_vel = finite_difference(joint_pos, dt)
|
| 448 |
+
body_lin_vel = finite_difference(body_pos, dt)
|
| 449 |
+
body_ang_vel = quaternion_angular_velocity(body_quat, dt)
|
| 450 |
+
|
| 451 |
+
write_csv(sample_dir / "joint_pos.csv", joint_pos, [f"joint_{i}" for i in range(29)])
|
| 452 |
+
write_csv(sample_dir / "joint_vel.csv", joint_vel, [f"joint_vel_{i}" for i in range(29)])
|
| 453 |
+
write_csv(sample_dir / "body_pos.csv", body_pos, ["body_0_x", "body_0_y", "body_0_z"])
|
| 454 |
+
write_csv(sample_dir / "body_quat.csv", body_quat, ["body_0_w", "body_0_x", "body_0_y", "body_0_z"])
|
| 455 |
+
write_csv(sample_dir / "body_lin_vel.csv", body_lin_vel, ["body_0_vel_x", "body_0_vel_y", "body_0_vel_z"])
|
| 456 |
+
write_csv(sample_dir / "body_ang_vel.csv", body_ang_vel, ["body_0_angvel_x", "body_0_angvel_y", "body_0_angvel_z"])
|
| 457 |
+
np.save(sample_dir / "motion.npy", pred_abs)
|
| 458 |
+
|
| 459 |
+
if args.export_delta:
|
| 460 |
+
write_csv(sample_dir / "joint_pos_delta.csv", pred_delta[:, :29], [f"joint_{i}" for i in range(29)])
|
| 461 |
+
write_csv(sample_dir / "body_pos_delta.csv", pred_delta[:, 29:32], ["body_0_x", "body_0_y", "body_0_z"])
|
| 462 |
+
write_csv(sample_dir / "body_quat_delta.csv", pred_delta[:, 32:36], ["body_0_w", "body_0_x", "body_0_y", "body_0_z"])
|
| 463 |
+
np.save(sample_dir / "motion_delta.npy", pred_delta)
|
| 464 |
+
|
| 465 |
+
write_metadata(sample_dir / "metadata.txt", motion_name, int(pred_abs.shape[0]))
|
| 466 |
+
write_teacher_info(
|
| 467 |
+
sample_dir / "info.txt",
|
| 468 |
+
source_motion_dir=motion_dir,
|
| 469 |
+
frames=int(pred_abs.shape[0]),
|
| 470 |
+
textop_actions=textop_actions,
|
| 471 |
+
num_past_frames=num_past_frames,
|
| 472 |
+
num_inference_steps=int(args.num_inference_steps),
|
| 473 |
+
metrics=metrics,
|
| 474 |
+
)
|
| 475 |
+
|
| 476 |
+
for src_name, dst_name in (("info.txt", "info_source.txt"), ("metadata.txt", "metadata_source.txt")):
|
| 477 |
+
src_path = motion_dir / src_name
|
| 478 |
+
if src_path.exists():
|
| 479 |
+
shutil.copy2(src_path, sample_dir / dst_name)
|
| 480 |
+
|
| 481 |
+
sample_meta = {
|
| 482 |
+
"run_dir": str(run_dir),
|
| 483 |
+
"ckpt_step": ckpt_step,
|
| 484 |
+
"source_motion_dir": str(motion_dir),
|
| 485 |
+
"output_dir": str(sample_dir),
|
| 486 |
+
"num_output_frames": int(pred_abs.shape[0]),
|
| 487 |
+
"textop_actions": textop_actions,
|
| 488 |
+
"decode_textop_joint_from_delta": decode_textop_joint_from_delta,
|
| 489 |
+
"num_past_frames": int(num_past_frames),
|
| 490 |
+
"num_inference_steps": int(args.num_inference_steps),
|
| 491 |
+
"metrics": metrics,
|
| 492 |
+
}
|
| 493 |
+
(sample_dir / "infer_meta.json").write_text(json.dumps(sample_meta, ensure_ascii=False, indent=2), encoding="utf-8")
|
| 494 |
+
(output_dir / "manifest.json").write_text(
|
| 495 |
+
json.dumps({"samples": [sample_meta]}, ensure_ascii=False, indent=2),
|
| 496 |
+
encoding="utf-8",
|
| 497 |
+
)
|
| 498 |
+
print(f"[OK] teacher reconstruction -> {sample_dir}")
|
| 499 |
+
if metrics:
|
| 500 |
+
print("[METRICS] " + json.dumps(metrics, ensure_ascii=False))
|
| 501 |
+
|
| 502 |
+
|
| 503 |
+
if __name__ == "__main__":
|
| 504 |
+
main()
|
scripts/infer/seed_text_to_motion.py
ADDED
|
@@ -0,0 +1,873 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#!/usr/bin/env python3
|
| 2 |
+
from __future__ import annotations
|
| 3 |
+
|
| 4 |
+
import argparse
|
| 5 |
+
import json
|
| 6 |
+
import math
|
| 7 |
+
import shutil
|
| 8 |
+
from pathlib import Path
|
| 9 |
+
from typing import List
|
| 10 |
+
|
| 11 |
+
import numpy as np
|
| 12 |
+
import pandas as pd
|
| 13 |
+
import torch
|
| 14 |
+
|
| 15 |
+
from psi.config.config import LaunchConfig
|
| 16 |
+
from psi.data.seed.motion_repr import (
|
| 17 |
+
SEED_ABS_DIM,
|
| 18 |
+
SEED_TEXTOP_ACTION_DIM,
|
| 19 |
+
TEXTOP_JOINT_DELTA,
|
| 20 |
+
TEXTOP_ROOT_DELTA_LOCAL,
|
| 21 |
+
TEXTOP_ROOT_HEIGHT,
|
| 22 |
+
TEXTOP_ROOT_ROT6D_DELTA,
|
| 23 |
+
quat_inv_wxyz,
|
| 24 |
+
quat_mul_wxyz,
|
| 25 |
+
build_seed_textop_state,
|
| 26 |
+
recover_seed_abs_from_textop_action,
|
| 27 |
+
rot6d_delta_to_quat,
|
| 28 |
+
world_delta_from_local,
|
| 29 |
+
)
|
| 30 |
+
from psi.data.seed.progress import build_progress_conditions
|
| 31 |
+
from psi.data.seed.text_prompt import (
|
| 32 |
+
DEFAULT_SEED_TEXT_FIELDS,
|
| 33 |
+
compose_seed_motion_prompt,
|
| 34 |
+
compose_seed_motion_prompt_from_row,
|
| 35 |
+
)
|
| 36 |
+
from psi.models.psi0 import Psi0Model
|
| 37 |
+
from psi.utils import parse_args_to_tyro_config, seed_everything
|
| 38 |
+
|
| 39 |
+
|
| 40 |
+
DEFAULT_TEXT_FIELDS: List[str] = list(DEFAULT_SEED_TEXT_FIELDS)
|
| 41 |
+
|
| 42 |
+
TARGET_FPS = 50
|
| 43 |
+
|
| 44 |
+
|
| 45 |
+
def compose_instruction(row: pd.Series, text_fields: List[str]) -> str:
|
| 46 |
+
return compose_seed_motion_prompt_from_row(row, text_fields)
|
| 47 |
+
|
| 48 |
+
|
| 49 |
+
def map_motion_dir(root_dir: Path, move_g1_path: str) -> Path:
|
| 50 |
+
rel = str(move_g1_path).strip().replace("\\", "/")
|
| 51 |
+
rel = rel.replace("g1/csv/", "g1/csv_sonic/")
|
| 52 |
+
rel = rel.replace("/csv/", "/csv_sonic/")
|
| 53 |
+
if rel.endswith(".csv"):
|
| 54 |
+
rel = rel[:-4]
|
| 55 |
+
|
| 56 |
+
if rel.startswith("dataset/seed/"):
|
| 57 |
+
rel = rel[len("dataset/seed/") :]
|
| 58 |
+
if rel.startswith("seed/"):
|
| 59 |
+
rel = rel[len("seed/") :]
|
| 60 |
+
|
| 61 |
+
p = Path(rel)
|
| 62 |
+
if p.is_absolute():
|
| 63 |
+
return p
|
| 64 |
+
return root_dir / p
|
| 65 |
+
|
| 66 |
+
|
| 67 |
+
def sanitize_motion_name(name: str) -> str:
|
| 68 |
+
out = "".join(ch if (ch.isalnum() or ch in {"_", "-", "."}) else "_" for ch in name.strip())
|
| 69 |
+
out = out.strip("._")
|
| 70 |
+
return out or "manual_prompt"
|
| 71 |
+
|
| 72 |
+
|
| 73 |
+
def normalize_quaternion_rows(motion_abs: np.ndarray) -> np.ndarray:
|
| 74 |
+
if motion_abs.shape[1] < 36:
|
| 75 |
+
return motion_abs
|
| 76 |
+
q = motion_abs[:, 32:36]
|
| 77 |
+
norm = np.linalg.norm(q, axis=1, keepdims=True)
|
| 78 |
+
norm = np.where(norm < 1e-8, 1.0, norm)
|
| 79 |
+
motion_abs[:, 32:36] = q / norm
|
| 80 |
+
return motion_abs
|
| 81 |
+
|
| 82 |
+
|
| 83 |
+
def quat_multiply(q1: np.ndarray, q2: np.ndarray) -> np.ndarray:
|
| 84 |
+
w1, x1, y1, z1 = q1
|
| 85 |
+
w2, x2, y2, z2 = q2
|
| 86 |
+
return np.array(
|
| 87 |
+
[
|
| 88 |
+
w1 * w2 - x1 * x2 - y1 * y2 - z1 * z2,
|
| 89 |
+
w1 * x2 + x1 * w2 + y1 * z2 - z1 * y2,
|
| 90 |
+
w1 * y2 - x1 * z2 + y1 * w2 + z1 * x2,
|
| 91 |
+
w1 * z2 + x1 * y2 - y1 * x2 + z1 * w2,
|
| 92 |
+
],
|
| 93 |
+
dtype=np.float64,
|
| 94 |
+
)
|
| 95 |
+
|
| 96 |
+
|
| 97 |
+
def quat_conjugate(q: np.ndarray) -> np.ndarray:
|
| 98 |
+
return np.array([q[0], -q[1], -q[2], -q[3]], dtype=np.float64)
|
| 99 |
+
|
| 100 |
+
|
| 101 |
+
def quat_to_rotvec(q: np.ndarray) -> np.ndarray:
|
| 102 |
+
q = q / max(np.linalg.norm(q), 1e-12)
|
| 103 |
+
if q[0] < 0.0:
|
| 104 |
+
q = -q
|
| 105 |
+
vec = q[1:]
|
| 106 |
+
sin_half = np.linalg.norm(vec)
|
| 107 |
+
if sin_half < 1e-10:
|
| 108 |
+
return np.zeros(3, dtype=np.float64)
|
| 109 |
+
axis = vec / sin_half
|
| 110 |
+
angle = 2.0 * np.arctan2(sin_half, q[0])
|
| 111 |
+
return axis * angle
|
| 112 |
+
|
| 113 |
+
|
| 114 |
+
def finite_difference(data: np.ndarray, dt: float) -> np.ndarray:
|
| 115 |
+
if data.shape[0] < 2:
|
| 116 |
+
return np.zeros_like(data, dtype=np.float64)
|
| 117 |
+
edge_order = 2 if data.shape[0] > 2 else 1
|
| 118 |
+
return np.gradient(data, dt, axis=0, edge_order=edge_order)
|
| 119 |
+
|
| 120 |
+
|
| 121 |
+
def quaternion_angular_velocity(quat_wxyz: np.ndarray, dt: float) -> np.ndarray:
|
| 122 |
+
ang_vel = np.zeros((quat_wxyz.shape[0], 3), dtype=np.float64)
|
| 123 |
+
if quat_wxyz.shape[0] < 2:
|
| 124 |
+
return ang_vel
|
| 125 |
+
|
| 126 |
+
for i in range(quat_wxyz.shape[0] - 1):
|
| 127 |
+
dq = quat_multiply(quat_wxyz[i + 1], quat_conjugate(quat_wxyz[i]))
|
| 128 |
+
ang_vel[i] = quat_to_rotvec(dq) / dt
|
| 129 |
+
|
| 130 |
+
ang_vel[-1] = ang_vel[-2]
|
| 131 |
+
return ang_vel
|
| 132 |
+
|
| 133 |
+
|
| 134 |
+
def write_csv(path: Path, array: np.ndarray, headers: List[str]) -> None:
|
| 135 |
+
np.savetxt(
|
| 136 |
+
path,
|
| 137 |
+
np.asarray(array, dtype=np.float64),
|
| 138 |
+
delimiter=",",
|
| 139 |
+
header=",".join(headers),
|
| 140 |
+
comments="",
|
| 141 |
+
fmt="%.6f",
|
| 142 |
+
)
|
| 143 |
+
|
| 144 |
+
|
| 145 |
+
def write_metadata(path: Path, motion_name: str, timesteps: int) -> None:
|
| 146 |
+
path.write_text(
|
| 147 |
+
"\n".join(
|
| 148 |
+
[
|
| 149 |
+
f"Metadata for: {motion_name}",
|
| 150 |
+
"==============================",
|
| 151 |
+
"",
|
| 152 |
+
"Body part indexes:",
|
| 153 |
+
"[0]",
|
| 154 |
+
"",
|
| 155 |
+
f"Total timesteps: {timesteps}",
|
| 156 |
+
"",
|
| 157 |
+
]
|
| 158 |
+
),
|
| 159 |
+
encoding="utf-8",
|
| 160 |
+
)
|
| 161 |
+
|
| 162 |
+
|
| 163 |
+
def write_info(
|
| 164 |
+
path: Path,
|
| 165 |
+
*,
|
| 166 |
+
source_path: str,
|
| 167 |
+
initial_pose_source: str,
|
| 168 |
+
frames: int,
|
| 169 |
+
use_delta_actions: bool,
|
| 170 |
+
root_delta_actions: bool,
|
| 171 |
+
textop_actions: bool,
|
| 172 |
+
num_past_frames: int,
|
| 173 |
+
enable_rtc: bool,
|
| 174 |
+
action_exec_horizon: int,
|
| 175 |
+
num_inference_steps: int,
|
| 176 |
+
) -> None:
|
| 177 |
+
lines = [
|
| 178 |
+
"Motion Information: seed_text_to_motion_inference",
|
| 179 |
+
"==================================================",
|
| 180 |
+
"",
|
| 181 |
+
f"source_path: {source_path}",
|
| 182 |
+
f"initial_pose_source: {initial_pose_source}",
|
| 183 |
+
f"target_fps: {TARGET_FPS}",
|
| 184 |
+
f"num_inference_steps: {num_inference_steps}",
|
| 185 |
+
f"trained_with_delta_actions: {use_delta_actions}",
|
| 186 |
+
f"trained_with_root_delta_actions: {root_delta_actions}",
|
| 187 |
+
f"trained_with_textop_actions: {textop_actions}",
|
| 188 |
+
f"num_past_frames: {num_past_frames}",
|
| 189 |
+
f"rtc_enabled: {enable_rtc}",
|
| 190 |
+
f"action_exec_horizon: {action_exec_horizon}",
|
| 191 |
+
"output_root_quaternion: wxyz",
|
| 192 |
+
"output_joint_angles: 29D",
|
| 193 |
+
"output_root_position: xyz",
|
| 194 |
+
"",
|
| 195 |
+
f"frames: {frames}",
|
| 196 |
+
"",
|
| 197 |
+
]
|
| 198 |
+
path.write_text("\n".join(lines), encoding="utf-8")
|
| 199 |
+
|
| 200 |
+
|
| 201 |
+
def load_pose_from_motion_dir(motion_dir: Path) -> np.ndarray:
|
| 202 |
+
joint = pd.read_csv(motion_dir / "joint_pos.csv").to_numpy(dtype=np.float32)
|
| 203 |
+
body_pos = pd.read_csv(motion_dir / "body_pos.csv").to_numpy(dtype=np.float32)
|
| 204 |
+
body_quat = pd.read_csv(motion_dir / "body_quat.csv").to_numpy(dtype=np.float32)
|
| 205 |
+
return np.concatenate([joint[0], body_pos[0], body_quat[0]], axis=0).astype(np.float32)
|
| 206 |
+
|
| 207 |
+
|
| 208 |
+
def load_pose_from_npy(path: Path) -> np.ndarray:
|
| 209 |
+
pose = np.load(path).astype(np.float32)
|
| 210 |
+
if pose.ndim == 2:
|
| 211 |
+
pose = pose[0]
|
| 212 |
+
if pose.shape != (36,):
|
| 213 |
+
raise ValueError(f"Expected init pose shape (36,) or (T,36), got {pose.shape} from {path}")
|
| 214 |
+
return pose.astype(np.float32)
|
| 215 |
+
|
| 216 |
+
|
| 217 |
+
def resolve_motion_dir(data_root: Path, motion_dir_arg: str) -> Path:
|
| 218 |
+
motion_dir = Path(motion_dir_arg).expanduser()
|
| 219 |
+
if motion_dir.is_absolute():
|
| 220 |
+
return motion_dir
|
| 221 |
+
return data_root / motion_dir
|
| 222 |
+
|
| 223 |
+
|
| 224 |
+
def load_initial_pose(root_dir: Path, row: pd.Series, restore_from_data: bool, strict: bool = False) -> np.ndarray:
|
| 225 |
+
if not restore_from_data:
|
| 226 |
+
if strict:
|
| 227 |
+
# <modify_jerk> root-delta也必须有真实首帧,否则root轨迹会从zero累加跑偏。
|
| 228 |
+
raise ValueError("Delta/root-delta inference requires a real initial pose. Do not use --no-restore-from-data.")
|
| 229 |
+
return np.zeros((36,), dtype=np.float32)
|
| 230 |
+
|
| 231 |
+
try:
|
| 232 |
+
motion_dir = map_motion_dir(root_dir, str(row["move_g1_path"]))
|
| 233 |
+
return load_pose_from_motion_dir(motion_dir)
|
| 234 |
+
except Exception as exc:
|
| 235 |
+
if strict:
|
| 236 |
+
raise RuntimeError(f"Failed to load initial pose from metadata row: {exc}") from exc
|
| 237 |
+
return np.zeros((36,), dtype=np.float32)
|
| 238 |
+
|
| 239 |
+
|
| 240 |
+
def load_source_frame_count(source_motion_dir: Path) -> int | None:
|
| 241 |
+
try:
|
| 242 |
+
joint = pd.read_csv(source_motion_dir / "joint_pos.csv")
|
| 243 |
+
frames = int(len(joint))
|
| 244 |
+
if frames > 0:
|
| 245 |
+
return frames
|
| 246 |
+
except Exception:
|
| 247 |
+
pass
|
| 248 |
+
return None
|
| 249 |
+
|
| 250 |
+
|
| 251 |
+
def build_model_states_from_abs(
|
| 252 |
+
state_history_abs: List[np.ndarray],
|
| 253 |
+
*,
|
| 254 |
+
textop_actions: bool,
|
| 255 |
+
odim: int,
|
| 256 |
+
) -> np.ndarray:
|
| 257 |
+
states_abs = np.stack(state_history_abs, axis=0).astype(np.float32)
|
| 258 |
+
states = build_seed_textop_state(states_abs[:, :SEED_ABS_DIM]) if textop_actions else states_abs
|
| 259 |
+
if states.shape[-1] < odim:
|
| 260 |
+
pad = np.zeros((states.shape[0], odim - states.shape[-1]), dtype=np.float32)
|
| 261 |
+
states = np.concatenate([states, pad], axis=-1)
|
| 262 |
+
if states.shape[-1] != odim:
|
| 263 |
+
raise ValueError(f"Infer state dim mismatch: built {states.shape[-1]}D, model odim={odim}")
|
| 264 |
+
return states.astype(np.float32)
|
| 265 |
+
|
| 266 |
+
|
| 267 |
+
def should_decode_textop_joint_from_delta(launch_config: LaunchConfig) -> bool:
|
| 268 |
+
if SEED_TEXTOP_ACTION_DIM == 39:
|
| 269 |
+
return False
|
| 270 |
+
group_weight = getattr(getattr(launch_config, "train", None), "loss_group_weight", None)
|
| 271 |
+
if group_weight is None:
|
| 272 |
+
return False
|
| 273 |
+
joint_abs_weight = float(getattr(group_weight, "joint_abs", 1.0))
|
| 274 |
+
joint_delta_weight = float(getattr(group_weight, "joint_delta", 1.0))
|
| 275 |
+
return joint_abs_weight == 0.0 and joint_delta_weight != 0.0
|
| 276 |
+
|
| 277 |
+
|
| 278 |
+
def recover_seed_abs_from_textop_action_with_joint_delta(
|
| 279 |
+
features: np.ndarray,
|
| 280 |
+
current_abs: np.ndarray,
|
| 281 |
+
) -> np.ndarray:
|
| 282 |
+
# breakpoint()
|
| 283 |
+
# 当joint_abs不参与训练时,推理侧也改成用joint_delta递推关节,避免使用未监督的joint_abs通道。
|
| 284 |
+
features = np.asarray(features, dtype=np.float32)
|
| 285 |
+
current_abs = np.asarray(current_abs, dtype=np.float32).copy()
|
| 286 |
+
if features.shape[-1] != SEED_TEXTOP_ACTION_DIM:
|
| 287 |
+
raise ValueError(f"Expected textop action dim {SEED_TEXTOP_ACTION_DIM}, got {features.shape[-1]}")
|
| 288 |
+
if current_abs.shape[-1] != SEED_ABS_DIM:
|
| 289 |
+
raise ValueError(f"Expected current_abs dim {SEED_ABS_DIM}, got {current_abs.shape[-1]}")
|
| 290 |
+
if TEXTOP_JOINT_DELTA.stop == TEXTOP_JOINT_DELTA.start:
|
| 291 |
+
return recover_seed_abs_from_textop_action(features, current_abs)
|
| 292 |
+
|
| 293 |
+
frames = []
|
| 294 |
+
prev_abs = current_abs.copy()
|
| 295 |
+
for feat in features:
|
| 296 |
+
joint_delta = feat[TEXTOP_JOINT_DELTA]
|
| 297 |
+
root_delta_local = feat[TEXTOP_ROOT_DELTA_LOCAL]
|
| 298 |
+
root_rot_delta = feat[TEXTOP_ROOT_ROT6D_DELTA]
|
| 299 |
+
root_height = feat[TEXTOP_ROOT_HEIGHT][0]
|
| 300 |
+
|
| 301 |
+
joint_abs = prev_abs[:29] + joint_delta
|
| 302 |
+
root_pos = prev_abs[29:32] + world_delta_from_local(prev_abs[32:36], root_delta_local)
|
| 303 |
+
root_pos[2] = root_height
|
| 304 |
+
rel_quat = rot6d_delta_to_quat(root_rot_delta)
|
| 305 |
+
root_quat = quat_mul_wxyz(prev_abs[32:36], rel_quat)
|
| 306 |
+
|
| 307 |
+
frame = np.concatenate([joint_abs, root_pos.astype(np.float32), root_quat.astype(np.float32)], axis=0)
|
| 308 |
+
frame = frame.astype(np.float32)
|
| 309 |
+
frames.append(frame)
|
| 310 |
+
prev_abs = frame
|
| 311 |
+
|
| 312 |
+
return np.stack(frames, axis=0).astype(np.float32)
|
| 313 |
+
|
| 314 |
+
|
| 315 |
+
def resolve_metadata_csv(data_root: Path, launch_config: LaunchConfig, metadata_csv_arg: str) -> Path | None:
|
| 316 |
+
if metadata_csv_arg:
|
| 317 |
+
metadata_csv = Path(metadata_csv_arg)
|
| 318 |
+
elif hasattr(launch_config.data, "metadata_csv"):
|
| 319 |
+
metadata_csv = Path(launch_config.data.metadata_csv)
|
| 320 |
+
elif hasattr(launch_config.data, "text"):
|
| 321 |
+
text_cfg = launch_config.data.text
|
| 322 |
+
metadata_csv_value = text_cfg.get("metadata_csv") if isinstance(text_cfg, dict) else getattr(text_cfg, "metadata_csv", None)
|
| 323 |
+
if metadata_csv_value is None:
|
| 324 |
+
return None
|
| 325 |
+
metadata_csv = Path(metadata_csv_value)
|
| 326 |
+
else:
|
| 327 |
+
return None
|
| 328 |
+
if not metadata_csv.is_absolute():
|
| 329 |
+
metadata_csv = data_root / metadata_csv
|
| 330 |
+
return metadata_csv
|
| 331 |
+
|
| 332 |
+
|
| 333 |
+
def resolve_text_data_root(launch_config: LaunchConfig) -> Path:
|
| 334 |
+
if hasattr(launch_config.data, "root_dir"):
|
| 335 |
+
return Path(launch_config.data.root_dir)
|
| 336 |
+
if hasattr(launch_config.data, "text"):
|
| 337 |
+
text_cfg = launch_config.data.text
|
| 338 |
+
root_dir = text_cfg.get("root_dir") if isinstance(text_cfg, dict) else getattr(text_cfg, "root_dir", None)
|
| 339 |
+
if root_dir:
|
| 340 |
+
return Path(root_dir)
|
| 341 |
+
raise AttributeError("run config has neither data.root_dir nor data.text.root_dir")
|
| 342 |
+
|
| 343 |
+
|
| 344 |
+
def load_prompt_initial_pose(
|
| 345 |
+
*,
|
| 346 |
+
args: argparse.Namespace,
|
| 347 |
+
data_root: Path,
|
| 348 |
+
metadata_csv: Path | None,
|
| 349 |
+
use_delta_actions: bool,
|
| 350 |
+
root_delta_actions: bool,
|
| 351 |
+
textop_actions: bool,
|
| 352 |
+
) -> tuple[np.ndarray, str]:
|
| 353 |
+
if args.init_pose_npy:
|
| 354 |
+
# <modify> 允许prompt模式显式指定36D初始姿态,避免delta模型从zero起步
|
| 355 |
+
init_path = Path(args.init_pose_npy).expanduser()
|
| 356 |
+
if not init_path.is_absolute():
|
| 357 |
+
cwd_path = Path.cwd() / init_path
|
| 358 |
+
data_path = data_root / init_path
|
| 359 |
+
init_path = cwd_path if cwd_path.exists() else data_path
|
| 360 |
+
return load_pose_from_npy(init_path), str(init_path)
|
| 361 |
+
|
| 362 |
+
if args.init_motion_dir:
|
| 363 |
+
# <modify> 允许prompt模式显式指定SEED动作目录,用该动作第一帧作为初始姿态
|
| 364 |
+
motion_dir = resolve_motion_dir(data_root, args.init_motion_dir)
|
| 365 |
+
return load_pose_from_motion_dir(motion_dir), str(motion_dir)
|
| 366 |
+
|
| 367 |
+
if args.restore_from_data:
|
| 368 |
+
# <modify> 默认从metadata的start-index动作取真实首帧,不再静默使用zero
|
| 369 |
+
if metadata_csv is None:
|
| 370 |
+
raise ValueError(
|
| 371 |
+
"Prompt inference needs an initial pose. Use --init-motion-dir or --init-pose-npy "
|
| 372 |
+
"when the run config has no metadata_csv."
|
| 373 |
+
)
|
| 374 |
+
prompt_metadata = pd.read_csv(metadata_csv)
|
| 375 |
+
if len(prompt_metadata) == 0:
|
| 376 |
+
raise ValueError(f"Empty metadata csv: {metadata_csv}")
|
| 377 |
+
ref_index = min(max(0, int(args.start_index)), len(prompt_metadata) - 1)
|
| 378 |
+
ref_row = prompt_metadata.iloc[ref_index]
|
| 379 |
+
motion_dir = map_motion_dir(data_root, str(ref_row["move_g1_path"]))
|
| 380 |
+
return load_pose_from_motion_dir(motion_dir), f"{motion_dir} (metadata index {ref_index})"
|
| 381 |
+
|
| 382 |
+
# <modify_jerk> hybrid root-delta prompt推理同样需要真实初始root。
|
| 383 |
+
if use_delta_actions or root_delta_actions or textop_actions:
|
| 384 |
+
raise ValueError(
|
| 385 |
+
"Delta/root-delta/TextOp prompt inference requires a real initial pose. "
|
| 386 |
+
"Use --init-motion-dir, --init-pose-npy, or keep --restore-from-data."
|
| 387 |
+
)
|
| 388 |
+
|
| 389 |
+
return np.zeros((36,), dtype=np.float32), "zero"
|
| 390 |
+
|
| 391 |
+
|
| 392 |
+
def rollout_motion(
|
| 393 |
+
*,
|
| 394 |
+
model: Psi0Model,
|
| 395 |
+
maxmin,
|
| 396 |
+
odim: int,
|
| 397 |
+
device: str,
|
| 398 |
+
instruction: str,
|
| 399 |
+
text_only: bool,
|
| 400 |
+
normalize_state: bool,
|
| 401 |
+
use_delta_actions: bool,
|
| 402 |
+
root_delta_actions: bool,
|
| 403 |
+
textop_actions: bool,
|
| 404 |
+
decode_textop_joint_from_delta: bool,
|
| 405 |
+
num_past_frames: int,
|
| 406 |
+
use_future_progress_embedding: bool,
|
| 407 |
+
enable_rtc: bool,
|
| 408 |
+
action_exec_horizon: int,
|
| 409 |
+
rtc_max_delay: int,
|
| 410 |
+
num_inference_steps: int,
|
| 411 |
+
target_frames: int,
|
| 412 |
+
init_abs_state: np.ndarray,
|
| 413 |
+
) -> tuple[np.ndarray, np.ndarray, int]:
|
| 414 |
+
horizon = int(model.action_horizon)
|
| 415 |
+
exec_horizon = int(action_exec_horizon) if enable_rtc else horizon
|
| 416 |
+
if exec_horizon <= 0 or exec_horizon > horizon:
|
| 417 |
+
raise ValueError(f"Invalid action_exec_horizon={exec_horizon}, action_horizon={horizon}")
|
| 418 |
+
rtc_delay = horizon - exec_horizon
|
| 419 |
+
if enable_rtc and rtc_delay > 0 and rtc_delay >= int(rtc_max_delay):
|
| 420 |
+
raise ValueError(
|
| 421 |
+
f"RTC requires action_horizon - action_exec_horizon < max_delay, "
|
| 422 |
+
f"got {horizon} - {exec_horizon} = {rtc_delay}, max_delay={rtc_max_delay}"
|
| 423 |
+
)
|
| 424 |
+
if target_frames <= 0:
|
| 425 |
+
target_frames = exec_horizon
|
| 426 |
+
# <modify> RTC推理按exec_horizon滚动;普通推理仍按完整action_horizon滚动
|
| 427 |
+
# 原逻辑:num_chunks = int(math.ceil(float(target_frames) / float(horizon)))
|
| 428 |
+
num_chunks = int(math.ceil(float(target_frames) / float(exec_horizon)))
|
| 429 |
+
|
| 430 |
+
current_abs = np.asarray(init_abs_state, dtype=np.float32).copy()
|
| 431 |
+
# <modify_states> 推理时维护和训练一致的多帧state历史,首个chunk用初始姿态重复填充。
|
| 432 |
+
state_horizon = int(num_past_frames) + 1
|
| 433 |
+
state_history: List[np.ndarray] = [current_abs.copy() for _ in range(state_horizon)]
|
| 434 |
+
all_abs: List[np.ndarray] = []
|
| 435 |
+
all_delta: List[np.ndarray] = []
|
| 436 |
+
prev_actions_norm: np.ndarray | None = None
|
| 437 |
+
remaining_frames = int(target_frames)
|
| 438 |
+
|
| 439 |
+
for _ in range(num_chunks):
|
| 440 |
+
generated_frames = int(target_frames - remaining_frames)
|
| 441 |
+
# <modify_states> 使用最近To帧state作为输入;保持shape(B, To, Ds)。
|
| 442 |
+
# 原逻辑:
|
| 443 |
+
# states = current_abs.reshape(1, odim).astype(np.float32)
|
| 444 |
+
states = build_model_states_from_abs(
|
| 445 |
+
state_history[-state_horizon:],
|
| 446 |
+
textop_actions=textop_actions,
|
| 447 |
+
odim=odim,
|
| 448 |
+
)
|
| 449 |
+
if normalize_state:
|
| 450 |
+
states = maxmin.normalize_state_func(states)
|
| 451 |
+
states_t = torch.from_numpy(states).unsqueeze(0).to(device)
|
| 452 |
+
future_progress_t = None
|
| 453 |
+
if use_future_progress_embedding:
|
| 454 |
+
_, future_progress = build_progress_conditions(
|
| 455 |
+
frame_id=generated_frames,
|
| 456 |
+
total_frames=target_frames,
|
| 457 |
+
action_horizon=horizon,
|
| 458 |
+
)
|
| 459 |
+
future_progress_t = torch.from_numpy(future_progress[None]).to(device)
|
| 460 |
+
|
| 461 |
+
with torch.inference_mode():
|
| 462 |
+
if enable_rtc and prev_actions_norm is not None and rtc_delay > 0:
|
| 463 |
+
# <modify> RTC推理把上一chunk尚未执行的尾部动作作为下一chunk的prefix条件
|
| 464 |
+
prev_rtc = np.concatenate(
|
| 465 |
+
[
|
| 466 |
+
prev_actions_norm[exec_horizon:, :],
|
| 467 |
+
np.zeros((exec_horizon, prev_actions_norm.shape[1]), dtype=np.float32),
|
| 468 |
+
],
|
| 469 |
+
axis=0,
|
| 470 |
+
)
|
| 471 |
+
prev_actions_t = torch.from_numpy(prev_rtc[None]).to(device)
|
| 472 |
+
pred_norm = model.predict_action_with_training_rtc_flow(
|
| 473 |
+
observations=None if text_only else [[]],
|
| 474 |
+
states=states_t,
|
| 475 |
+
instructions=[instruction],
|
| 476 |
+
num_inference_steps=num_inference_steps,
|
| 477 |
+
traj2ds=None,
|
| 478 |
+
prev_actions=prev_actions_t,
|
| 479 |
+
inference_delay=rtc_delay,
|
| 480 |
+
max_delay=int(rtc_max_delay),
|
| 481 |
+
future_progress=future_progress_t,
|
| 482 |
+
)[0].float().cpu().numpy()
|
| 483 |
+
else:
|
| 484 |
+
pred_norm = model.predict_action(
|
| 485 |
+
observations=None if text_only else [[]],
|
| 486 |
+
states=states_t,
|
| 487 |
+
instructions=[instruction],
|
| 488 |
+
num_inference_steps=num_inference_steps,
|
| 489 |
+
traj2ds=None,
|
| 490 |
+
future_progress=future_progress_t,
|
| 491 |
+
)[0].float().cpu().numpy()
|
| 492 |
+
|
| 493 |
+
# <modify> 训练侧ActionStateTransform会把normalized action裁剪到[-1,1],推理反归一化前也要保持同一范围。
|
| 494 |
+
# 原逻辑:chunk_pred = np.asarray(maxmin.denormalize(pred_norm), dtype=np.float32)
|
| 495 |
+
pred_norm_clipped = np.clip(np.asarray(pred_norm, dtype=np.float32), -1.0, 1.0)
|
| 496 |
+
chunk_pred = np.asarray(maxmin.denormalize(pred_norm_clipped), dtype=np.float32)
|
| 497 |
+
|
| 498 |
+
# <modify_textop_repr> TextOp训练目标先恢复到36D绝对姿态;root旋转delta在helper里按相对旋转6D解码。
|
| 499 |
+
if textop_actions:
|
| 500 |
+
if decode_textop_joint_from_delta:
|
| 501 |
+
chunk_abs = recover_seed_abs_from_textop_action_with_joint_delta(chunk_pred, current_abs)
|
| 502 |
+
else:
|
| 503 |
+
chunk_abs = recover_seed_abs_from_textop_action(chunk_pred, current_abs)
|
| 504 |
+
prev_abs = np.concatenate([current_abs[None, :], chunk_abs[:-1]], axis=0)
|
| 505 |
+
joint_delta = chunk_abs[:, :29] - prev_abs[:, :29]
|
| 506 |
+
root_pos_delta = chunk_abs[:, 29:32] - prev_abs[:, 29:32]
|
| 507 |
+
root_quat_delta = quat_mul_wxyz(quat_inv_wxyz(prev_abs[:, 32:36]), chunk_abs[:, 32:36])
|
| 508 |
+
chunk_delta = np.concatenate([joint_delta, root_pos_delta, root_quat_delta], axis=1)
|
| 509 |
+
elif use_delta_actions:
|
| 510 |
+
# <modify> delta action需要从current_abs累加成绝对姿态,支持多chunk滚动
|
| 511 |
+
# 原逻辑:把chunk_pred直接当绝对姿态,和delta训练目标不一致
|
| 512 |
+
chunk_delta = chunk_pred
|
| 513 |
+
chunk_abs = np.cumsum(chunk_delta, axis=0) + current_abs[None, :]
|
| 514 |
+
elif root_delta_actions:
|
| 515 |
+
# <modify_jerk> hybrid action:关节直接用绝对预测,root 7维从当前姿态逐帧累加。
|
| 516 |
+
chunk_abs = chunk_pred.copy()
|
| 517 |
+
root_abs = current_abs[29:36].astype(np.float32).copy()
|
| 518 |
+
root_abs_seq = []
|
| 519 |
+
for root_delta in chunk_pred[:, 29:36]:
|
| 520 |
+
root_abs = root_abs + root_delta
|
| 521 |
+
quat_norm = np.linalg.norm(root_abs[3:7])
|
| 522 |
+
if quat_norm > 1e-8:
|
| 523 |
+
root_abs[3:7] = root_abs[3:7] / quat_norm
|
| 524 |
+
root_abs_seq.append(root_abs.copy())
|
| 525 |
+
chunk_abs[:, 29:36] = np.stack(root_abs_seq, axis=0)
|
| 526 |
+
|
| 527 |
+
first_joint_delta = chunk_abs[0, :29] - current_abs[:29]
|
| 528 |
+
if chunk_abs.shape[0] > 1:
|
| 529 |
+
rest_joint_delta = chunk_abs[1:, :29] - chunk_abs[:-1, :29]
|
| 530 |
+
joint_delta = np.concatenate([first_joint_delta[None, :], rest_joint_delta], axis=0)
|
| 531 |
+
else:
|
| 532 |
+
joint_delta = first_joint_delta[None, :]
|
| 533 |
+
chunk_delta = np.concatenate([joint_delta, chunk_pred[:, 29:36]], axis=1)
|
| 534 |
+
else:
|
| 535 |
+
chunk_abs = chunk_pred
|
| 536 |
+
first_delta = chunk_abs[0] - current_abs
|
| 537 |
+
if chunk_abs.shape[0] > 1:
|
| 538 |
+
rest_delta = chunk_abs[1:] - chunk_abs[:-1]
|
| 539 |
+
chunk_delta = np.concatenate([first_delta[None, :], rest_delta], axis=0)
|
| 540 |
+
else:
|
| 541 |
+
chunk_delta = first_delta[None, :]
|
| 542 |
+
|
| 543 |
+
chunk_abs = normalize_quaternion_rows(chunk_abs.astype(np.float32))
|
| 544 |
+
# <modify> TextOp且joint_abs未监督时,RTC prefix不能继续使用模型预测的joint_abs通道。
|
| 545 |
+
# 原逻辑:prev_actions_norm = np.asarray(pred_norm, dtype=np.float32).copy()
|
| 546 |
+
# 这里用joint_delta还原后的关节绝对值回填0:29,再重新normalize,避免坏joint_abs污染下一chunk。
|
| 547 |
+
if textop_actions and decode_textop_joint_from_delta:
|
| 548 |
+
chunk_pred_for_rtc = chunk_pred.copy()
|
| 549 |
+
chunk_pred_for_rtc[:, :29] = chunk_abs[:, :29]
|
| 550 |
+
# <modify> 这里不能直接调用maxmin.normalize,因为当前配置normalize_state=True时会要求data里有states。
|
| 551 |
+
action_min = np.asarray(maxmin.action_min, dtype=np.float32)
|
| 552 |
+
action_max = np.asarray(maxmin.action_max, dtype=np.float32)
|
| 553 |
+
ill_mask = np.abs(action_max - action_min) < 1e-4 * (np.abs(action_max) + np.abs(action_min) + 1e-8)
|
| 554 |
+
action_max = action_max.copy()
|
| 555 |
+
action_max[ill_mask] = 1.0
|
| 556 |
+
chunk_pred_for_rtc_norm = np.where(
|
| 557 |
+
ill_mask,
|
| 558 |
+
chunk_pred_for_rtc,
|
| 559 |
+
(chunk_pred_for_rtc - action_min) / (action_max - action_min) * 2 - 1,
|
| 560 |
+
)
|
| 561 |
+
if bool(getattr(maxmin, "use_norm_mask", False)):
|
| 562 |
+
action_norm_masks = np.asarray(maxmin.action_norm_masks, dtype=bool)
|
| 563 |
+
chunk_pred_for_rtc_norm = np.where(action_norm_masks, chunk_pred_for_rtc_norm, chunk_pred_for_rtc)
|
| 564 |
+
prev_actions_norm = np.clip(chunk_pred_for_rtc_norm, -1, 1).astype(np.float32).copy()
|
| 565 |
+
else:
|
| 566 |
+
# <modify> RTC prefix也使用裁剪后的normalized action,避免把越界预测作为clean prefix条件。
|
| 567 |
+
# 原逻辑:prev_actions_norm = np.asarray(pred_norm, dtype=np.float32).copy()
|
| 568 |
+
prev_actions_norm = pred_norm_clipped.copy()
|
| 569 |
+
# <modify> RTC只导出本轮实际执行的前exec_horizon帧;普通模式exec_horizon等于horizon
|
| 570 |
+
take = min(exec_horizon, remaining_frames)
|
| 571 |
+
if take <= 0:
|
| 572 |
+
break
|
| 573 |
+
executed_abs = chunk_abs[:take]
|
| 574 |
+
executed_delta = chunk_delta[:take].astype(np.float32)
|
| 575 |
+
current_abs = executed_abs[-1].copy()
|
| 576 |
+
# <modify_states> 每执行一步都推进state历史,下一chunk看到的是真实rollout历史而不是单帧。
|
| 577 |
+
for frame_abs in executed_abs:
|
| 578 |
+
state_history.append(frame_abs.astype(np.float32).copy())
|
| 579 |
+
state_history = state_history[-state_horizon:]
|
| 580 |
+
all_abs.append(executed_abs)
|
| 581 |
+
all_delta.append(executed_delta)
|
| 582 |
+
remaining_frames -= take
|
| 583 |
+
|
| 584 |
+
pred_abs = np.concatenate(all_abs, axis=0)[:target_frames]
|
| 585 |
+
pred_delta = np.concatenate(all_delta, axis=0)[:target_frames]
|
| 586 |
+
pred_abs = normalize_quaternion_rows(pred_abs.astype(np.float32))
|
| 587 |
+
return pred_abs, pred_delta, num_chunks
|
| 588 |
+
|
| 589 |
+
|
| 590 |
+
def resolve_ckpt_step(run_dir: Path, ckpt_step: str) -> int | str:
|
| 591 |
+
if ckpt_step == "best":
|
| 592 |
+
# <modify> 支持--ckpt-step best,直接读取checkpoints/best_ckpt目录。
|
| 593 |
+
best_ckpt = run_dir / "checkpoints" / "best_ckpt"
|
| 594 |
+
if not best_ckpt.exists():
|
| 595 |
+
raise RuntimeError(f"No best_ckpt found under {run_dir / 'checkpoints'}")
|
| 596 |
+
return "best_ckpt"
|
| 597 |
+
|
| 598 |
+
if ckpt_step != "latest":
|
| 599 |
+
return int(ckpt_step)
|
| 600 |
+
|
| 601 |
+
ckpt_root = run_dir / "checkpoints"
|
| 602 |
+
candidates = []
|
| 603 |
+
for p in ckpt_root.iterdir():
|
| 604 |
+
if p.is_dir() and p.name.startswith("ckpt_"):
|
| 605 |
+
try:
|
| 606 |
+
candidates.append(int(p.name.split("_")[-1]))
|
| 607 |
+
except ValueError:
|
| 608 |
+
continue
|
| 609 |
+
if not candidates:
|
| 610 |
+
raise RuntimeError(f"No ckpt_* found under {ckpt_root}")
|
| 611 |
+
return max(candidates)
|
| 612 |
+
|
| 613 |
+
|
| 614 |
+
def parse_args() -> argparse.Namespace:
|
| 615 |
+
parser = argparse.ArgumentParser(description="Seed text->motion offline inference exporter")
|
| 616 |
+
parser.add_argument("--run-dir", type=str, required=True)
|
| 617 |
+
parser.add_argument("--ckpt-step", type=str, default="latest", help="integer step, 'latest', or 'best'")
|
| 618 |
+
parser.add_argument("--output-dir", type=str, required=True)
|
| 619 |
+
parser.add_argument("--num-samples", type=int, default=1)
|
| 620 |
+
parser.add_argument("--start-index", type=int, default=0)
|
| 621 |
+
parser.add_argument("--num-inference-steps", type=int, default=10)
|
| 622 |
+
parser.add_argument("--device", type=str, default="cuda:0")
|
| 623 |
+
parser.add_argument("--metadata-csv", type=str, default="")
|
| 624 |
+
parser.add_argument("--prompt", type=str, default="", help="Direct text prompt for manual inference")
|
| 625 |
+
parser.add_argument("--prompt-motion-name", type=str, default="manual_prompt")
|
| 626 |
+
parser.add_argument("--init-motion-dir", type=str, default="", help="SEED csv_sonic motion dir used for prompt initial pose")
|
| 627 |
+
parser.add_argument("--init-pose-npy", type=str, default="", help="36D pose npy, or Tx36 motion npy, used for prompt initial pose")
|
| 628 |
+
parser.add_argument("--num-output-frames", type=int, default=0, help="0 means auto length")
|
| 629 |
+
parser.add_argument("--seed", type=int, default=42)
|
| 630 |
+
parser.add_argument("--restore-from-data", action="store_true", default=True)
|
| 631 |
+
parser.add_argument("--no-restore-from-data", dest="restore_from_data", action="store_false")
|
| 632 |
+
parser.add_argument("--export-delta", action="store_true", default=False)
|
| 633 |
+
parser.add_argument("--disable-rtc", action="store_true", default=False)
|
| 634 |
+
parser.add_argument(
|
| 635 |
+
"--force-text-only",
|
| 636 |
+
action="store_true",
|
| 637 |
+
default=False,
|
| 638 |
+
help="Use the same checkpoint but force text-only VLM inputs, even if run_config is audio-only.",
|
| 639 |
+
)
|
| 640 |
+
return parser.parse_args()
|
| 641 |
+
|
| 642 |
+
|
| 643 |
+
def main() -> None:
|
| 644 |
+
args = parse_args()
|
| 645 |
+
|
| 646 |
+
run_dir = Path(args.run_dir)
|
| 647 |
+
output_dir = Path(args.output_dir)
|
| 648 |
+
output_dir.mkdir(parents=True, exist_ok=True)
|
| 649 |
+
|
| 650 |
+
cfg_template: LaunchConfig = parse_args_to_tyro_config(run_dir / "argv.txt") # type: ignore
|
| 651 |
+
launch_config = cfg_template.model_validate_json((run_dir / "run_config.json").read_text())
|
| 652 |
+
|
| 653 |
+
seed_everything(launch_config.seed or args.seed)
|
| 654 |
+
|
| 655 |
+
# 找到指定ckpt
|
| 656 |
+
ckpt_step = resolve_ckpt_step(run_dir, args.ckpt_step)
|
| 657 |
+
model = Psi0Model.from_pretrained(run_dir, ckpt_step, launch_config, device=args.device)
|
| 658 |
+
if args.force_text_only:
|
| 659 |
+
model.text_only = True
|
| 660 |
+
model.audio_only = False
|
| 661 |
+
model.video_only = False
|
| 662 |
+
model.to(args.device)
|
| 663 |
+
model.eval()
|
| 664 |
+
|
| 665 |
+
data_root = resolve_text_data_root(launch_config)
|
| 666 |
+
prompt_mode = bool(args.prompt.strip())
|
| 667 |
+
use_delta_actions = bool(getattr(launch_config.data.transform.repack, "use_delta_actions", False))
|
| 668 |
+
# <modify_jerk> run_config里记录hybrid action模式:关节绝对,root delta。
|
| 669 |
+
root_delta_actions = bool(getattr(launch_config.data.transform.repack, "root_delta_actions", False))
|
| 670 |
+
# TextOp模式下训练目标是39D action/state;推理端用同一个decoder恢复为36D绝对动作。
|
| 671 |
+
textop_actions = bool(getattr(launch_config.data.transform.repack, "textop_actions", False))
|
| 672 |
+
decode_textop_joint_from_delta = textop_actions and should_decode_textop_joint_from_delta(launch_config)
|
| 673 |
+
# <modify_states> 从run_config读取SEED多帧state配置;旧checkpoint默认退化为单帧state。
|
| 674 |
+
num_past_frames = int(getattr(launch_config.data.transform.repack, "num_past_frames", 0))
|
| 675 |
+
# <modify> run_config里model.rtc=True时默认使用RTC滚动推理;可用--disable-rtc关闭
|
| 676 |
+
enable_rtc = bool(getattr(launch_config.model, "rtc", False)) and (not args.disable_rtc)
|
| 677 |
+
use_future_progress_embedding = bool(getattr(launch_config.model, "use_future_progress_embedding", False))
|
| 678 |
+
action_exec_horizon = int(getattr(launch_config.model, "action_exec_horizon", int(model.action_horizon)))
|
| 679 |
+
rtc_max_delay = int(getattr(launch_config.model, "max_delay", 0))
|
| 680 |
+
metadata_csv = resolve_metadata_csv(data_root, launch_config, args.metadata_csv)
|
| 681 |
+
|
| 682 |
+
metadata = None
|
| 683 |
+
if not prompt_mode:
|
| 684 |
+
metadata = pd.read_csv(metadata_csv)
|
| 685 |
+
start = max(0, args.start_index)
|
| 686 |
+
end = min(len(metadata), start + max(0, args.num_samples))
|
| 687 |
+
if start >= end:
|
| 688 |
+
raise ValueError(f"Empty sampling range: start={start}, end={end}, metadata_size={len(metadata)}")
|
| 689 |
+
sample_indices = list(range(start, end))
|
| 690 |
+
else:
|
| 691 |
+
sample_indices = [0]
|
| 692 |
+
|
| 693 |
+
text_only = bool(args.force_text_only) or bool(getattr(launch_config.data.transform.model, "text_only", False))
|
| 694 |
+
maxmin = launch_config.data.transform.field
|
| 695 |
+
odim = int(launch_config.model.odim)
|
| 696 |
+
action_dim = int(getattr(launch_config.model, "action_dim", getattr(model, "action_dim", SEED_TEXTOP_ACTION_DIM)))
|
| 697 |
+
if textop_actions and (action_dim != SEED_TEXTOP_ACTION_DIM or odim != SEED_TEXTOP_ACTION_DIM):
|
| 698 |
+
raise ValueError(
|
| 699 |
+
f"TextOp infer/train dim mismatch: action_dim={action_dim}, odim={odim}, expected={SEED_TEXTOP_ACTION_DIM}"
|
| 700 |
+
)
|
| 701 |
+
|
| 702 |
+
manifest = {
|
| 703 |
+
"run_dir": str(run_dir),
|
| 704 |
+
"ckpt_step": ckpt_step,
|
| 705 |
+
"metadata_csv": str(metadata_csv) if metadata_csv is not None else "",
|
| 706 |
+
"mode": "prompt" if prompt_mode else "metadata",
|
| 707 |
+
"text_only": text_only,
|
| 708 |
+
"action_dim": int(action_dim),
|
| 709 |
+
"state_dim": int(odim),
|
| 710 |
+
"use_delta_actions": use_delta_actions,
|
| 711 |
+
"root_delta_actions": root_delta_actions,
|
| 712 |
+
"textop_actions": textop_actions,
|
| 713 |
+
"decode_textop_joint_from_delta": decode_textop_joint_from_delta,
|
| 714 |
+
"num_past_frames": int(num_past_frames),
|
| 715 |
+
"use_future_progress_embedding": bool(use_future_progress_embedding),
|
| 716 |
+
"enable_rtc": enable_rtc,
|
| 717 |
+
"action_exec_horizon": int(action_exec_horizon),
|
| 718 |
+
"rtc_max_delay": int(rtc_max_delay),
|
| 719 |
+
"num_inference_steps": int(args.num_inference_steps),
|
| 720 |
+
"num_output_frames": int(args.num_output_frames),
|
| 721 |
+
"export_delta": bool(args.export_delta),
|
| 722 |
+
"samples": [],
|
| 723 |
+
}
|
| 724 |
+
|
| 725 |
+
for idx in sample_indices:
|
| 726 |
+
if prompt_mode:
|
| 727 |
+
raw_instruction = args.prompt.strip()
|
| 728 |
+
instruction = compose_seed_motion_prompt(raw_instruction)
|
| 729 |
+
source_motion_dir = Path("manual_prompt")
|
| 730 |
+
# <modify> prompt模式下delta推理不能再用zero初始姿态,默认从metadata的start-index动作取真实首帧
|
| 731 |
+
# 原逻辑:x0 = np.zeros((36,), dtype=np.float32)
|
| 732 |
+
x0, initial_pose_source = load_prompt_initial_pose(
|
| 733 |
+
args=args,
|
| 734 |
+
data_root=data_root,
|
| 735 |
+
metadata_csv=metadata_csv,
|
| 736 |
+
use_delta_actions=use_delta_actions,
|
| 737 |
+
root_delta_actions=root_delta_actions,
|
| 738 |
+
textop_actions=textop_actions,
|
| 739 |
+
)
|
| 740 |
+
auto_frames = max(int(model.action_horizon), 300)
|
| 741 |
+
target_frames = int(args.num_output_frames) if args.num_output_frames > 0 else auto_frames
|
| 742 |
+
motion_name = sanitize_motion_name(args.prompt_motion_name)
|
| 743 |
+
sample_dir = output_dir / "g1" / "csv_sonic" / "manual" / motion_name
|
| 744 |
+
else:
|
| 745 |
+
assert metadata is not None
|
| 746 |
+
row = metadata.iloc[idx]
|
| 747 |
+
raw_instruction = ""
|
| 748 |
+
instruction = compose_instruction(row, DEFAULT_TEXT_FIELDS)
|
| 749 |
+
source_motion_dir = map_motion_dir(data_root, str(row.get("move_g1_path", "")))
|
| 750 |
+
# <modify> delta推理加载初始姿态失败时直接报错,避免静默退回zero
|
| 751 |
+
x0 = load_initial_pose(
|
| 752 |
+
data_root,
|
| 753 |
+
row,
|
| 754 |
+
restore_from_data=args.restore_from_data,
|
| 755 |
+
strict=(use_delta_actions or root_delta_actions or textop_actions),
|
| 756 |
+
)
|
| 757 |
+
initial_pose_source = str(source_motion_dir) if args.restore_from_data else "zero"
|
| 758 |
+
source_frames = load_source_frame_count(source_motion_dir)
|
| 759 |
+
if args.num_output_frames > 0:
|
| 760 |
+
target_frames = int(args.num_output_frames)
|
| 761 |
+
elif source_frames is not None:
|
| 762 |
+
target_frames = source_frames
|
| 763 |
+
else:
|
| 764 |
+
target_frames = int(model.action_horizon)
|
| 765 |
+
# Output folder keeps the same relative hierarchy as csv_sonic
|
| 766 |
+
try:
|
| 767 |
+
rel_dir = source_motion_dir.relative_to(data_root)
|
| 768 |
+
except Exception:
|
| 769 |
+
rel_dir = Path(f"sample_{idx:04d}")
|
| 770 |
+
sample_dir = output_dir / rel_dir
|
| 771 |
+
motion_name = source_motion_dir.name if source_motion_dir.name else f"sample_{idx:04d}"
|
| 772 |
+
|
| 773 |
+
pred_abs, pred_delta, rollout_chunks = rollout_motion(
|
| 774 |
+
model=model,
|
| 775 |
+
maxmin=maxmin,
|
| 776 |
+
odim=odim,
|
| 777 |
+
device=args.device,
|
| 778 |
+
instruction=instruction,
|
| 779 |
+
text_only=text_only,
|
| 780 |
+
normalize_state=bool(getattr(maxmin, "normalize_state", False)),
|
| 781 |
+
use_delta_actions=use_delta_actions,
|
| 782 |
+
root_delta_actions=root_delta_actions,
|
| 783 |
+
textop_actions=textop_actions,
|
| 784 |
+
decode_textop_joint_from_delta=decode_textop_joint_from_delta,
|
| 785 |
+
num_past_frames=num_past_frames,
|
| 786 |
+
use_future_progress_embedding=use_future_progress_embedding,
|
| 787 |
+
enable_rtc=enable_rtc,
|
| 788 |
+
action_exec_horizon=action_exec_horizon,
|
| 789 |
+
rtc_max_delay=rtc_max_delay,
|
| 790 |
+
num_inference_steps=int(args.num_inference_steps),
|
| 791 |
+
target_frames=target_frames,
|
| 792 |
+
init_abs_state=x0,
|
| 793 |
+
)
|
| 794 |
+
|
| 795 |
+
sample_dir.mkdir(parents=True, exist_ok=True)
|
| 796 |
+
|
| 797 |
+
joint_pos = pred_abs[:, :29]
|
| 798 |
+
body_pos = pred_abs[:, 29:32]
|
| 799 |
+
body_quat = pred_abs[:, 32:36]
|
| 800 |
+
|
| 801 |
+
dt = 1.0 / float(TARGET_FPS)
|
| 802 |
+
joint_vel = finite_difference(joint_pos, dt)
|
| 803 |
+
body_lin_vel = finite_difference(body_pos, dt)
|
| 804 |
+
body_ang_vel = quaternion_angular_velocity(body_quat, dt)
|
| 805 |
+
|
| 806 |
+
# Main outputs: match csv_sonic naming/headers
|
| 807 |
+
write_csv(sample_dir / "joint_pos.csv", joint_pos, [f"joint_{i}" for i in range(29)])
|
| 808 |
+
write_csv(sample_dir / "joint_vel.csv", joint_vel, [f"joint_vel_{i}" for i in range(29)])
|
| 809 |
+
write_csv(sample_dir / "body_pos.csv", body_pos, ["body_0_x", "body_0_y", "body_0_z"])
|
| 810 |
+
write_csv(sample_dir / "body_quat.csv", body_quat, ["body_0_w", "body_0_x", "body_0_y", "body_0_z"])
|
| 811 |
+
write_csv(sample_dir / "body_lin_vel.csv", body_lin_vel, ["body_0_vel_x", "body_0_vel_y", "body_0_vel_z"])
|
| 812 |
+
write_csv(sample_dir / "body_ang_vel.csv", body_ang_vel, ["body_0_angvel_x", "body_0_angvel_y", "body_0_angvel_z"])
|
| 813 |
+
|
| 814 |
+
np.save(sample_dir / "motion.npy", pred_abs)
|
| 815 |
+
|
| 816 |
+
# Optional delta outputs
|
| 817 |
+
if args.export_delta:
|
| 818 |
+
write_csv(sample_dir / "joint_pos_delta.csv", pred_delta[:, :29], [f"joint_{i}" for i in range(29)])
|
| 819 |
+
write_csv(sample_dir / "body_pos_delta.csv", pred_delta[:, 29:32], ["body_0_x", "body_0_y", "body_0_z"])
|
| 820 |
+
write_csv(sample_dir / "body_quat_delta.csv", pred_delta[:, 32:36], ["body_0_w", "body_0_x", "body_0_y", "body_0_z"])
|
| 821 |
+
np.save(sample_dir / "motion_delta.npy", pred_delta)
|
| 822 |
+
|
| 823 |
+
write_metadata(sample_dir / "metadata.txt", motion_name, int(pred_abs.shape[0]))
|
| 824 |
+
write_info(
|
| 825 |
+
sample_dir / "info.txt",
|
| 826 |
+
source_path=str(source_motion_dir),
|
| 827 |
+
initial_pose_source=initial_pose_source,
|
| 828 |
+
frames=int(pred_abs.shape[0]),
|
| 829 |
+
use_delta_actions=use_delta_actions,
|
| 830 |
+
root_delta_actions=root_delta_actions,
|
| 831 |
+
textop_actions=textop_actions,
|
| 832 |
+
num_past_frames=int(num_past_frames),
|
| 833 |
+
enable_rtc=enable_rtc,
|
| 834 |
+
action_exec_horizon=int(action_exec_horizon),
|
| 835 |
+
num_inference_steps=int(args.num_inference_steps),
|
| 836 |
+
)
|
| 837 |
+
|
| 838 |
+
# Keep source references if existing
|
| 839 |
+
if not prompt_mode:
|
| 840 |
+
src_info = source_motion_dir / "info.txt"
|
| 841 |
+
src_meta = source_motion_dir / "metadata.txt"
|
| 842 |
+
if src_info.exists():
|
| 843 |
+
shutil.copy2(src_info, sample_dir / "info_source.txt")
|
| 844 |
+
if src_meta.exists():
|
| 845 |
+
shutil.copy2(src_meta, sample_dir / "metadata_source.txt")
|
| 846 |
+
|
| 847 |
+
sample_meta = {
|
| 848 |
+
"sample_index": int(idx),
|
| 849 |
+
"instruction": instruction,
|
| 850 |
+
"raw_instruction": raw_instruction,
|
| 851 |
+
"move_g1_path": "" if prompt_mode else str(metadata.iloc[idx].get("move_g1_path", "")),
|
| 852 |
+
"source_motion_dir": str(source_motion_dir),
|
| 853 |
+
"initial_pose_source": initial_pose_source,
|
| 854 |
+
"num_past_frames": int(num_past_frames),
|
| 855 |
+
"root_delta_actions": root_delta_actions,
|
| 856 |
+
"textop_actions": textop_actions,
|
| 857 |
+
"decode_textop_joint_from_delta": decode_textop_joint_from_delta,
|
| 858 |
+
"enable_rtc": enable_rtc,
|
| 859 |
+
"action_exec_horizon": int(action_exec_horizon),
|
| 860 |
+
"output_dir": str(sample_dir),
|
| 861 |
+
"num_output_frames": int(pred_abs.shape[0]),
|
| 862 |
+
"rollout_chunks": int(rollout_chunks),
|
| 863 |
+
}
|
| 864 |
+
(sample_dir / "infer_meta.json").write_text(json.dumps(sample_meta, ensure_ascii=False, indent=2))
|
| 865 |
+
manifest["samples"].append(sample_meta)
|
| 866 |
+
print(f"[OK] sample={idx} -> {sample_dir}")
|
| 867 |
+
|
| 868 |
+
(output_dir / "manifest.json").write_text(json.dumps(manifest, ensure_ascii=False, indent=2))
|
| 869 |
+
print(f"[DONE] wrote manifest: {output_dir / 'manifest.json'}")
|
| 870 |
+
|
| 871 |
+
|
| 872 |
+
if __name__ == "__main__":
|
| 873 |
+
main()
|
scripts/infer/seed_video_to_motion.py
ADDED
|
@@ -0,0 +1,548 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#!/usr/bin/env python3
|
| 2 |
+
from __future__ import annotations
|
| 3 |
+
|
| 4 |
+
import argparse
|
| 5 |
+
import json
|
| 6 |
+
import math
|
| 7 |
+
import shutil
|
| 8 |
+
from pathlib import Path
|
| 9 |
+
from typing import List
|
| 10 |
+
|
| 11 |
+
import numpy as np
|
| 12 |
+
import torch
|
| 13 |
+
|
| 14 |
+
from psi.config.config import LaunchConfig
|
| 15 |
+
from psi.data.seed.motion_repr import quat_inv_wxyz, quat_mul_wxyz, recover_seed_abs_from_textop_action
|
| 16 |
+
from psi.data.seed.progress import build_progress_conditions
|
| 17 |
+
from psi.data.seed.seed_video_motion_dataset import (
|
| 18 |
+
_probe_video,
|
| 19 |
+
load_video_array,
|
| 20 |
+
load_video_clip_array,
|
| 21 |
+
load_video_embed_cache,
|
| 22 |
+
video_embed_cache_path,
|
| 23 |
+
)
|
| 24 |
+
from psi.models.motion_distill import MotionDistillPsi0Model
|
| 25 |
+
from psi.models.psi0 import Psi0Model
|
| 26 |
+
from psi.utils import parse_args_to_tyro_config, seed_everything
|
| 27 |
+
|
| 28 |
+
from seed_audio_to_motion import resolve_motion_dir, resolve_video_path
|
| 29 |
+
from seed_text_to_motion import (
|
| 30 |
+
TARGET_FPS,
|
| 31 |
+
build_model_states_from_abs,
|
| 32 |
+
finite_difference,
|
| 33 |
+
load_pose_from_motion_dir,
|
| 34 |
+
load_source_frame_count,
|
| 35 |
+
normalize_quaternion_rows,
|
| 36 |
+
quaternion_angular_velocity,
|
| 37 |
+
recover_seed_abs_from_textop_action_with_joint_delta,
|
| 38 |
+
resolve_ckpt_step,
|
| 39 |
+
sanitize_motion_name,
|
| 40 |
+
should_decode_textop_joint_from_delta,
|
| 41 |
+
write_csv,
|
| 42 |
+
write_metadata,
|
| 43 |
+
)
|
| 44 |
+
|
| 45 |
+
|
| 46 |
+
def cfg_get(obj, key: str, default=None):
|
| 47 |
+
if isinstance(obj, dict):
|
| 48 |
+
return obj.get(key, default)
|
| 49 |
+
return getattr(obj, key, default)
|
| 50 |
+
|
| 51 |
+
|
| 52 |
+
def video_data_cfg(launch_config: LaunchConfig, config_key: str = "video"):
|
| 53 |
+
return cfg_get(launch_config.data, config_key, cfg_get(launch_config.data, "video", launch_config.data))
|
| 54 |
+
|
| 55 |
+
|
| 56 |
+
def rollout_video_motion(
|
| 57 |
+
*,
|
| 58 |
+
model: Psi0Model,
|
| 59 |
+
maxmin,
|
| 60 |
+
odim: int,
|
| 61 |
+
device: str,
|
| 62 |
+
video: np.ndarray | None,
|
| 63 |
+
video_embed: dict | None,
|
| 64 |
+
video_path: Path,
|
| 65 |
+
instruction: str,
|
| 66 |
+
video_size_pixels: int,
|
| 67 |
+
video_sample_fps: float,
|
| 68 |
+
video_decoder: str,
|
| 69 |
+
video_pixel_cache_dtype: str,
|
| 70 |
+
video_clip_mode: bool,
|
| 71 |
+
video_clip_motion_fps: float,
|
| 72 |
+
normalize_state: bool,
|
| 73 |
+
textop_actions: bool,
|
| 74 |
+
decode_textop_joint_from_delta: bool,
|
| 75 |
+
num_past_frames: int,
|
| 76 |
+
use_future_progress_embedding: bool,
|
| 77 |
+
enable_rtc: bool,
|
| 78 |
+
action_exec_horizon: int,
|
| 79 |
+
rtc_max_delay: int,
|
| 80 |
+
num_inference_steps: int,
|
| 81 |
+
target_frames: int,
|
| 82 |
+
init_abs_state: np.ndarray,
|
| 83 |
+
) -> tuple[np.ndarray, np.ndarray, int]:
|
| 84 |
+
horizon = int(model.action_horizon)
|
| 85 |
+
exec_horizon = int(action_exec_horizon) if enable_rtc else horizon
|
| 86 |
+
if exec_horizon <= 0 or exec_horizon > horizon:
|
| 87 |
+
raise ValueError(f"Invalid action_exec_horizon={exec_horizon}, action_horizon={horizon}")
|
| 88 |
+
rtc_delay = horizon - exec_horizon
|
| 89 |
+
if enable_rtc and rtc_delay > 0 and rtc_delay >= int(rtc_max_delay):
|
| 90 |
+
raise ValueError(
|
| 91 |
+
f"RTC requires action_horizon - action_exec_horizon < max_delay, "
|
| 92 |
+
f"got {horizon} - {exec_horizon} = {rtc_delay}, max_delay={rtc_max_delay}"
|
| 93 |
+
)
|
| 94 |
+
if not textop_actions:
|
| 95 |
+
raise ValueError("This video inference path is for the 39D TextOp action checkpoint.")
|
| 96 |
+
if target_frames <= 0:
|
| 97 |
+
target_frames = exec_horizon
|
| 98 |
+
|
| 99 |
+
num_chunks = int(math.ceil(float(target_frames) / float(exec_horizon)))
|
| 100 |
+
current_abs = np.asarray(init_abs_state, dtype=np.float32).copy()
|
| 101 |
+
state_horizon = int(num_past_frames) + 1
|
| 102 |
+
state_history: List[np.ndarray] = [current_abs.copy() for _ in range(state_horizon)]
|
| 103 |
+
all_abs: List[np.ndarray] = []
|
| 104 |
+
all_delta: List[np.ndarray] = []
|
| 105 |
+
prev_actions_norm: np.ndarray | None = None
|
| 106 |
+
remaining_frames = int(target_frames)
|
| 107 |
+
|
| 108 |
+
for _ in range(num_chunks):
|
| 109 |
+
if video_clip_mode:
|
| 110 |
+
chunk_index = len(all_abs)
|
| 111 |
+
video_clip_start_sec = float(chunk_index * exec_horizon) / float(video_clip_motion_fps)
|
| 112 |
+
video_clip_duration_sec = float(horizon) / float(video_clip_motion_fps)
|
| 113 |
+
chunk_video, _, _ = load_video_clip_array(
|
| 114 |
+
video_path,
|
| 115 |
+
start_sec=video_clip_start_sec,
|
| 116 |
+
duration_sec=video_clip_duration_sec,
|
| 117 |
+
video_sample_fps=video_sample_fps,
|
| 118 |
+
video_size_pixels=video_size_pixels,
|
| 119 |
+
video_decoder=video_decoder,
|
| 120 |
+
)
|
| 121 |
+
chunk_video_embed = None
|
| 122 |
+
else:
|
| 123 |
+
chunk_video = video
|
| 124 |
+
chunk_video_embed = video_embed
|
| 125 |
+
|
| 126 |
+
frame_cursor = target_frames - remaining_frames
|
| 127 |
+
states = build_model_states_from_abs(
|
| 128 |
+
state_history[-state_horizon:],
|
| 129 |
+
textop_actions=textop_actions,
|
| 130 |
+
odim=odim,
|
| 131 |
+
)
|
| 132 |
+
if normalize_state:
|
| 133 |
+
states = maxmin.normalize_state_func(states)
|
| 134 |
+
states_t = torch.from_numpy(states).unsqueeze(0).to(device)
|
| 135 |
+
future_progress_t = None
|
| 136 |
+
if use_future_progress_embedding:
|
| 137 |
+
if video_clip_mode:
|
| 138 |
+
# Training video_clip samples are cropped to one local horizon:
|
| 139 |
+
# state frame 0 plus action frames 1..horizon. Keep progress local
|
| 140 |
+
# at inference so action-token positions match the training distribution.
|
| 141 |
+
progress_frame_id = 0
|
| 142 |
+
progress_total_frames = horizon + 1
|
| 143 |
+
else:
|
| 144 |
+
progress_frame_id = frame_cursor
|
| 145 |
+
progress_total_frames = target_frames
|
| 146 |
+
_, future_progress = build_progress_conditions(
|
| 147 |
+
frame_id=progress_frame_id,
|
| 148 |
+
total_frames=progress_total_frames,
|
| 149 |
+
action_horizon=horizon,
|
| 150 |
+
)
|
| 151 |
+
future_progress_t = torch.from_numpy(future_progress[None]).to(device)
|
| 152 |
+
|
| 153 |
+
with torch.inference_mode():
|
| 154 |
+
if enable_rtc and prev_actions_norm is not None and rtc_delay > 0:
|
| 155 |
+
prev_rtc = np.concatenate(
|
| 156 |
+
[
|
| 157 |
+
prev_actions_norm[exec_horizon:, :],
|
| 158 |
+
np.zeros((exec_horizon, prev_actions_norm.shape[1]), dtype=np.float32),
|
| 159 |
+
],
|
| 160 |
+
axis=0,
|
| 161 |
+
)
|
| 162 |
+
prev_actions_t = torch.from_numpy(prev_rtc[None]).to(device)
|
| 163 |
+
pred_norm = model.predict_action_with_training_rtc_flow(
|
| 164 |
+
observations=None,
|
| 165 |
+
states=states_t,
|
| 166 |
+
instructions=[instruction],
|
| 167 |
+
num_inference_steps=num_inference_steps,
|
| 168 |
+
traj2ds=None,
|
| 169 |
+
prev_actions=prev_actions_t,
|
| 170 |
+
inference_delay=rtc_delay,
|
| 171 |
+
max_delay=int(rtc_max_delay),
|
| 172 |
+
videos=[chunk_video] if chunk_video_embed is None else None,
|
| 173 |
+
video_embeds=[chunk_video_embed] if chunk_video_embed is not None else None,
|
| 174 |
+
video_paths=[str(video_path)],
|
| 175 |
+
video_size_pixels=int(video_size_pixels),
|
| 176 |
+
video_sample_fps=float(video_sample_fps),
|
| 177 |
+
video_pixel_cache_dtype=video_pixel_cache_dtype,
|
| 178 |
+
future_progress=future_progress_t,
|
| 179 |
+
)[0].float().cpu().numpy()
|
| 180 |
+
else:
|
| 181 |
+
pred_norm = model.predict_action(
|
| 182 |
+
observations=None,
|
| 183 |
+
states=states_t,
|
| 184 |
+
instructions=[instruction],
|
| 185 |
+
num_inference_steps=num_inference_steps,
|
| 186 |
+
traj2ds=None,
|
| 187 |
+
videos=[chunk_video] if chunk_video_embed is None else None,
|
| 188 |
+
video_embeds=[chunk_video_embed] if chunk_video_embed is not None else None,
|
| 189 |
+
video_paths=[str(video_path)],
|
| 190 |
+
video_size_pixels=int(video_size_pixels),
|
| 191 |
+
video_sample_fps=float(video_sample_fps),
|
| 192 |
+
video_pixel_cache_dtype=video_pixel_cache_dtype,
|
| 193 |
+
future_progress=future_progress_t,
|
| 194 |
+
)[0].float().cpu().numpy()
|
| 195 |
+
|
| 196 |
+
pred_norm_clipped = np.clip(np.asarray(pred_norm, dtype=np.float32), -1.0, 1.0)
|
| 197 |
+
chunk_pred = np.asarray(maxmin.denormalize(pred_norm_clipped), dtype=np.float32)
|
| 198 |
+
|
| 199 |
+
if decode_textop_joint_from_delta:
|
| 200 |
+
chunk_abs = recover_seed_abs_from_textop_action_with_joint_delta(chunk_pred, current_abs)
|
| 201 |
+
else:
|
| 202 |
+
chunk_abs = recover_seed_abs_from_textop_action(chunk_pred, current_abs)
|
| 203 |
+
|
| 204 |
+
prev_abs = np.concatenate([current_abs[None, :], chunk_abs[:-1]], axis=0)
|
| 205 |
+
joint_delta = chunk_abs[:, :29] - prev_abs[:, :29]
|
| 206 |
+
root_pos_delta = chunk_abs[:, 29:32] - prev_abs[:, 29:32]
|
| 207 |
+
root_quat_delta = quat_mul_wxyz(quat_inv_wxyz(prev_abs[:, 32:36]), chunk_abs[:, 32:36])
|
| 208 |
+
chunk_delta = np.concatenate([joint_delta, root_pos_delta, root_quat_delta], axis=1)
|
| 209 |
+
|
| 210 |
+
chunk_abs = normalize_quaternion_rows(chunk_abs.astype(np.float32))
|
| 211 |
+
if decode_textop_joint_from_delta:
|
| 212 |
+
chunk_pred_for_rtc = chunk_pred.copy()
|
| 213 |
+
chunk_pred_for_rtc[:, :29] = chunk_abs[:, :29]
|
| 214 |
+
from seed_audio_to_motion import normalize_action_for_rtc
|
| 215 |
+
prev_actions_norm = normalize_action_for_rtc(maxmin, chunk_pred_for_rtc)
|
| 216 |
+
else:
|
| 217 |
+
prev_actions_norm = pred_norm_clipped.copy()
|
| 218 |
+
|
| 219 |
+
take = min(exec_horizon, remaining_frames)
|
| 220 |
+
if take <= 0:
|
| 221 |
+
break
|
| 222 |
+
executed_abs = chunk_abs[:take]
|
| 223 |
+
executed_delta = chunk_delta[:take].astype(np.float32)
|
| 224 |
+
current_abs = executed_abs[-1].copy()
|
| 225 |
+
for frame_abs in executed_abs:
|
| 226 |
+
state_history.append(frame_abs.astype(np.float32).copy())
|
| 227 |
+
state_history = state_history[-state_horizon:]
|
| 228 |
+
all_abs.append(executed_abs)
|
| 229 |
+
all_delta.append(executed_delta)
|
| 230 |
+
remaining_frames -= take
|
| 231 |
+
|
| 232 |
+
pred_abs = np.concatenate(all_abs, axis=0)[:target_frames]
|
| 233 |
+
pred_delta = np.concatenate(all_delta, axis=0)[:target_frames]
|
| 234 |
+
pred_abs = normalize_quaternion_rows(pred_abs.astype(np.float32))
|
| 235 |
+
return pred_abs, pred_delta, num_chunks
|
| 236 |
+
|
| 237 |
+
|
| 238 |
+
def write_video_info(
|
| 239 |
+
path: Path,
|
| 240 |
+
*,
|
| 241 |
+
video_decoder: str,
|
| 242 |
+
source_motion_dir: Path,
|
| 243 |
+
video_path: Path,
|
| 244 |
+
instruction: str,
|
| 245 |
+
initial_pose_source: str,
|
| 246 |
+
frames: int,
|
| 247 |
+
textop_actions: bool,
|
| 248 |
+
num_past_frames: int,
|
| 249 |
+
enable_rtc: bool,
|
| 250 |
+
action_exec_horizon: int,
|
| 251 |
+
num_inference_steps: int,
|
| 252 |
+
video_sample_fps: float,
|
| 253 |
+
video_size_pixels: int,
|
| 254 |
+
video_pixel_cache_dtype: str,
|
| 255 |
+
video_embed_path: str,
|
| 256 |
+
source_video_fps: float,
|
| 257 |
+
video_duration_sec: float,
|
| 258 |
+
full_video: bool,
|
| 259 |
+
video_config_key: str,
|
| 260 |
+
video_clip_motion_fps: float,
|
| 261 |
+
) -> None:
|
| 262 |
+
lines = [
|
| 263 |
+
"Motion Information: seed_video_to_motion_inference",
|
| 264 |
+
"===================================================",
|
| 265 |
+
"",
|
| 266 |
+
f"video_decoder: {video_decoder}",
|
| 267 |
+
f"video_config_key: {video_config_key}",
|
| 268 |
+
f"source_motion_dir: {source_motion_dir}",
|
| 269 |
+
f"video_path: {video_path}",
|
| 270 |
+
f"instruction: {instruction}",
|
| 271 |
+
f"initial_pose_source: {initial_pose_source}",
|
| 272 |
+
f"target_fps: {TARGET_FPS}",
|
| 273 |
+
f"num_inference_steps: {num_inference_steps}",
|
| 274 |
+
f"trained_with_textop_actions: {textop_actions}",
|
| 275 |
+
f"num_past_frames: {num_past_frames}",
|
| 276 |
+
f"rtc_enabled: {enable_rtc}",
|
| 277 |
+
f"action_exec_horizon: {action_exec_horizon}",
|
| 278 |
+
f"video_sample_fps: {video_sample_fps}",
|
| 279 |
+
f"video_size_pixels: {video_size_pixels}",
|
| 280 |
+
f"video_pixel_cache_dtype: {video_pixel_cache_dtype}",
|
| 281 |
+
f"video_embed_path: {video_embed_path}",
|
| 282 |
+
f"source_video_fps: {source_video_fps}",
|
| 283 |
+
f"video_duration_sec: {video_duration_sec}",
|
| 284 |
+
f"full_video: {full_video}",
|
| 285 |
+
f"video_clip_motion_fps: {video_clip_motion_fps}",
|
| 286 |
+
"use_audio_in_video: False",
|
| 287 |
+
"output_root_quaternion: wxyz",
|
| 288 |
+
"output_joint_angles: 29D",
|
| 289 |
+
"output_root_position: xyz",
|
| 290 |
+
"",
|
| 291 |
+
f"frames: {frames}",
|
| 292 |
+
"",
|
| 293 |
+
]
|
| 294 |
+
path.write_text("\n".join(lines), encoding="utf-8")
|
| 295 |
+
|
| 296 |
+
|
| 297 |
+
def parse_args() -> argparse.Namespace:
|
| 298 |
+
parser = argparse.ArgumentParser(description="Seed video->motion offline inference exporter")
|
| 299 |
+
parser.add_argument("--run-dir", type=str, required=True)
|
| 300 |
+
parser.add_argument("--ckpt-step", type=str, default="latest", help="integer step, 'latest', or 'best'")
|
| 301 |
+
parser.add_argument("--output-dir", type=str, required=True)
|
| 302 |
+
parser.add_argument("--motion-dir", type=str, required=True)
|
| 303 |
+
parser.add_argument("--video-path", type=str, default="")
|
| 304 |
+
parser.add_argument("--num-output-frames", type=int, default=0, help="0 means source motion length")
|
| 305 |
+
parser.add_argument("--num-inference-steps", type=int, default=10)
|
| 306 |
+
parser.add_argument("--device", type=str, default="cuda:0")
|
| 307 |
+
parser.add_argument("--seed", type=int, default=42)
|
| 308 |
+
parser.add_argument("--prompt", type=str, default="", help="Optional text instruction paired with the video")
|
| 309 |
+
parser.add_argument("--export-delta", action="store_true", default=False)
|
| 310 |
+
parser.add_argument("--disable-rtc", action="store_true", default=False)
|
| 311 |
+
parser.add_argument("--video-config-key", type=str, default="video", help="data config key: video or video_clip")
|
| 312 |
+
parser.add_argument("--video-clip", action="store_true", default=False, help="Use one aligned video clip per rollout chunk")
|
| 313 |
+
parser.add_argument(
|
| 314 |
+
"--video-pixel-cache-dtype",
|
| 315 |
+
choices=("auto", "none", "fp16", "bf16"),
|
| 316 |
+
default="auto",
|
| 317 |
+
help="Match cached Qwen pixel-value quantization; auto uses fp16 for cached video_clip runs.",
|
| 318 |
+
)
|
| 319 |
+
parser.add_argument(
|
| 320 |
+
"--video-clip-motion-fps",
|
| 321 |
+
type=float,
|
| 322 |
+
default=0.0,
|
| 323 |
+
help="Override motion fps used to align rollout chunks to video clips; <=0 infers it from source motion/video.",
|
| 324 |
+
)
|
| 325 |
+
return parser.parse_args()
|
| 326 |
+
|
| 327 |
+
|
| 328 |
+
def main() -> None:
|
| 329 |
+
args = parse_args()
|
| 330 |
+
|
| 331 |
+
run_dir = Path(args.run_dir)
|
| 332 |
+
output_dir = Path(args.output_dir)
|
| 333 |
+
output_dir.mkdir(parents=True, exist_ok=True)
|
| 334 |
+
|
| 335 |
+
cfg_template: LaunchConfig = parse_args_to_tyro_config(run_dir / "argv.txt") # type: ignore
|
| 336 |
+
launch_config = cfg_template.model_validate_json((run_dir / "run_config.json").read_text())
|
| 337 |
+
seed_everything(launch_config.seed or args.seed)
|
| 338 |
+
|
| 339 |
+
ckpt_step = resolve_ckpt_step(run_dir, args.ckpt_step)
|
| 340 |
+
model_cls = (
|
| 341 |
+
MotionDistillPsi0Model
|
| 342 |
+
if hasattr(launch_config.model, "motion_distill_stage")
|
| 343 |
+
else Psi0Model
|
| 344 |
+
)
|
| 345 |
+
model = model_cls.from_pretrained(run_dir, ckpt_step, launch_config, device=args.device)
|
| 346 |
+
model.text_only = False
|
| 347 |
+
model.audio_only = False
|
| 348 |
+
model.video_only = True
|
| 349 |
+
model.to(args.device)
|
| 350 |
+
model.eval()
|
| 351 |
+
|
| 352 |
+
video_cfg = video_data_cfg(launch_config, args.video_config_key)
|
| 353 |
+
data_root = Path(cfg_get(video_cfg, "root_dir"))
|
| 354 |
+
video_root = Path(cfg_get(video_cfg, "video_dir", ""))
|
| 355 |
+
motion_dir = resolve_motion_dir(data_root, args.motion_dir)
|
| 356 |
+
video_path = resolve_video_path(video_root, motion_dir, args.video_path)
|
| 357 |
+
if not motion_dir.exists():
|
| 358 |
+
raise FileNotFoundError(f"motion_dir does not exist: {motion_dir}")
|
| 359 |
+
if not video_path.exists():
|
| 360 |
+
raise FileNotFoundError(f"video_path does not exist: {video_path}")
|
| 361 |
+
|
| 362 |
+
maxmin = launch_config.data.transform.field
|
| 363 |
+
odim = int(launch_config.model.odim)
|
| 364 |
+
textop_actions = bool(getattr(launch_config.data.transform.repack, "textop_actions", False))
|
| 365 |
+
decode_textop_joint_from_delta = textop_actions and should_decode_textop_joint_from_delta(launch_config)
|
| 366 |
+
num_past_frames = int(getattr(launch_config.data.transform.repack, "num_past_frames", 0))
|
| 367 |
+
use_future_progress_embedding = bool(getattr(launch_config.model, "use_future_progress_embedding", False))
|
| 368 |
+
enable_rtc = bool(getattr(launch_config.model, "rtc", False)) and (not args.disable_rtc)
|
| 369 |
+
action_exec_horizon = int(getattr(launch_config.model, "action_exec_horizon", int(model.action_horizon)))
|
| 370 |
+
rtc_max_delay = int(getattr(launch_config.model, "max_delay", 0))
|
| 371 |
+
video_sample_fps = float(cfg_get(video_cfg, "video_sample_fps", 1.0))
|
| 372 |
+
video_size_pixels = int(cfg_get(video_cfg, "video_size_pixels", 50176))
|
| 373 |
+
video_decoder = str(cfg_get(video_cfg, "video_decoder", "opencv"))
|
| 374 |
+
if args.video_pixel_cache_dtype == "auto":
|
| 375 |
+
video_pixel_cache_dtype = (
|
| 376 |
+
"fp16"
|
| 377 |
+
if args.video_clip and bool(cfg_get(video_cfg, "use_omni_hidden_cache", False))
|
| 378 |
+
else "none"
|
| 379 |
+
)
|
| 380 |
+
else:
|
| 381 |
+
video_pixel_cache_dtype = str(args.video_pixel_cache_dtype)
|
| 382 |
+
video_cfg_motion_fps = float(cfg_get(video_cfg, "motion_fps", TARGET_FPS))
|
| 383 |
+
video_embed_dir = "" if args.video_clip else str(cfg_get(video_cfg, "video_embed_dir", ""))
|
| 384 |
+
instruction = args.prompt.strip()
|
| 385 |
+
|
| 386 |
+
video_embed = None
|
| 387 |
+
video_embed_path = ""
|
| 388 |
+
if (not args.video_clip) and video_embed_dir.strip():
|
| 389 |
+
cache_path = video_embed_cache_path(
|
| 390 |
+
video_path,
|
| 391 |
+
video_embed_dir,
|
| 392 |
+
video_sample_fps=video_sample_fps,
|
| 393 |
+
video_size_pixels=video_size_pixels,
|
| 394 |
+
)
|
| 395 |
+
if cache_path.exists():
|
| 396 |
+
video_embed = load_video_embed_cache(cache_path)
|
| 397 |
+
video_embed_path = str(cache_path)
|
| 398 |
+
|
| 399 |
+
if args.video_clip:
|
| 400 |
+
_, _, source_video_fps, video_duration_sec = _probe_video(video_path)
|
| 401 |
+
video = None
|
| 402 |
+
elif video_embed is None:
|
| 403 |
+
video, source_video_fps, video_duration_sec = load_video_array(
|
| 404 |
+
video_path,
|
| 405 |
+
video_sample_fps=video_sample_fps,
|
| 406 |
+
video_size_pixels=video_size_pixels,
|
| 407 |
+
video_decoder=video_decoder,
|
| 408 |
+
)
|
| 409 |
+
else:
|
| 410 |
+
video = None
|
| 411 |
+
source_video_fps = float(video_embed["source_video_fps"])
|
| 412 |
+
video_duration_sec = float(video_embed["video_duration_sec"])
|
| 413 |
+
|
| 414 |
+
x0 = load_pose_from_motion_dir(motion_dir)
|
| 415 |
+
source_frames = load_source_frame_count(motion_dir)
|
| 416 |
+
target_frames = int(args.num_output_frames) if args.num_output_frames > 0 else int(source_frames or 0)
|
| 417 |
+
if args.video_clip and float(args.video_clip_motion_fps) > 0:
|
| 418 |
+
video_clip_motion_fps = float(args.video_clip_motion_fps)
|
| 419 |
+
elif args.video_clip and video_cfg_motion_fps > 0:
|
| 420 |
+
video_clip_motion_fps = float(video_cfg_motion_fps)
|
| 421 |
+
elif args.video_clip and source_frames > 0 and video_duration_sec > 0:
|
| 422 |
+
video_clip_motion_fps = float(source_frames) / float(video_duration_sec)
|
| 423 |
+
else:
|
| 424 |
+
video_clip_motion_fps = float(TARGET_FPS)
|
| 425 |
+
|
| 426 |
+
pred_abs, pred_delta, rollout_chunks = rollout_video_motion(
|
| 427 |
+
model=model,
|
| 428 |
+
maxmin=maxmin,
|
| 429 |
+
odim=odim,
|
| 430 |
+
device=args.device,
|
| 431 |
+
video=video,
|
| 432 |
+
video_embed=video_embed,
|
| 433 |
+
video_path=video_path,
|
| 434 |
+
instruction=instruction,
|
| 435 |
+
video_size_pixels=video_size_pixels,
|
| 436 |
+
video_sample_fps=video_sample_fps,
|
| 437 |
+
video_decoder=video_decoder,
|
| 438 |
+
video_pixel_cache_dtype=video_pixel_cache_dtype,
|
| 439 |
+
video_clip_mode=bool(args.video_clip),
|
| 440 |
+
video_clip_motion_fps=float(video_clip_motion_fps),
|
| 441 |
+
normalize_state=bool(getattr(maxmin, "normalize_state", False)),
|
| 442 |
+
textop_actions=textop_actions,
|
| 443 |
+
decode_textop_joint_from_delta=decode_textop_joint_from_delta,
|
| 444 |
+
num_past_frames=num_past_frames,
|
| 445 |
+
use_future_progress_embedding=use_future_progress_embedding,
|
| 446 |
+
enable_rtc=enable_rtc,
|
| 447 |
+
action_exec_horizon=action_exec_horizon,
|
| 448 |
+
rtc_max_delay=rtc_max_delay,
|
| 449 |
+
num_inference_steps=int(args.num_inference_steps),
|
| 450 |
+
target_frames=target_frames,
|
| 451 |
+
init_abs_state=x0,
|
| 452 |
+
)
|
| 453 |
+
|
| 454 |
+
motion_name = sanitize_motion_name(motion_dir.name)
|
| 455 |
+
sample_dir = output_dir / motion_name
|
| 456 |
+
sample_dir.mkdir(parents=True, exist_ok=True)
|
| 457 |
+
|
| 458 |
+
joint_pos = pred_abs[:, :29]
|
| 459 |
+
body_pos = pred_abs[:, 29:32]
|
| 460 |
+
body_quat = pred_abs[:, 32:36]
|
| 461 |
+
|
| 462 |
+
dt = 1.0 / float(TARGET_FPS)
|
| 463 |
+
joint_vel = finite_difference(joint_pos, dt)
|
| 464 |
+
body_lin_vel = finite_difference(body_pos, dt)
|
| 465 |
+
body_ang_vel = quaternion_angular_velocity(body_quat, dt)
|
| 466 |
+
|
| 467 |
+
write_csv(sample_dir / "joint_pos.csv", joint_pos, [f"joint_{i}" for i in range(29)])
|
| 468 |
+
write_csv(sample_dir / "joint_vel.csv", joint_vel, [f"joint_vel_{i}" for i in range(29)])
|
| 469 |
+
write_csv(sample_dir / "body_pos.csv", body_pos, ["body_0_x", "body_0_y", "body_0_z"])
|
| 470 |
+
write_csv(sample_dir / "body_quat.csv", body_quat, ["body_0_w", "body_0_x", "body_0_y", "body_0_z"])
|
| 471 |
+
write_csv(sample_dir / "body_lin_vel.csv", body_lin_vel, ["body_0_vel_x", "body_0_vel_y", "body_0_vel_z"])
|
| 472 |
+
write_csv(sample_dir / "body_ang_vel.csv", body_ang_vel, ["body_0_angvel_x", "body_0_angvel_y", "body_0_angvel_z"])
|
| 473 |
+
np.save(sample_dir / "motion.npy", pred_abs)
|
| 474 |
+
|
| 475 |
+
if args.export_delta:
|
| 476 |
+
write_csv(sample_dir / "joint_pos_delta.csv", pred_delta[:, :29], [f"joint_{i}" for i in range(29)])
|
| 477 |
+
write_csv(sample_dir / "body_pos_delta.csv", pred_delta[:, 29:32], ["body_0_x", "body_0_y", "body_0_z"])
|
| 478 |
+
write_csv(sample_dir / "body_quat_delta.csv", pred_delta[:, 32:36], ["body_0_w", "body_0_x", "body_0_y", "body_0_z"])
|
| 479 |
+
np.save(sample_dir / "motion_delta.npy", pred_delta)
|
| 480 |
+
|
| 481 |
+
write_metadata(sample_dir / "metadata.txt", motion_name, int(pred_abs.shape[0]))
|
| 482 |
+
write_video_info(
|
| 483 |
+
sample_dir / "info.txt",
|
| 484 |
+
video_decoder=video_decoder,
|
| 485 |
+
source_motion_dir=motion_dir,
|
| 486 |
+
video_path=video_path,
|
| 487 |
+
instruction=instruction,
|
| 488 |
+
initial_pose_source=str(motion_dir),
|
| 489 |
+
frames=int(pred_abs.shape[0]),
|
| 490 |
+
textop_actions=textop_actions,
|
| 491 |
+
num_past_frames=int(num_past_frames),
|
| 492 |
+
enable_rtc=enable_rtc,
|
| 493 |
+
action_exec_horizon=int(action_exec_horizon),
|
| 494 |
+
num_inference_steps=int(args.num_inference_steps),
|
| 495 |
+
video_sample_fps=float(video_sample_fps),
|
| 496 |
+
video_size_pixels=int(video_size_pixels),
|
| 497 |
+
video_pixel_cache_dtype=video_pixel_cache_dtype,
|
| 498 |
+
video_embed_path=video_embed_path,
|
| 499 |
+
source_video_fps=float(source_video_fps),
|
| 500 |
+
video_duration_sec=float(video_duration_sec),
|
| 501 |
+
full_video=not bool(args.video_clip),
|
| 502 |
+
video_config_key=str(args.video_config_key),
|
| 503 |
+
video_clip_motion_fps=float(video_clip_motion_fps),
|
| 504 |
+
)
|
| 505 |
+
|
| 506 |
+
for src_name, dst_name in (("info.txt", "info_source.txt"), ("metadata.txt", "metadata_source.txt")):
|
| 507 |
+
src_path = motion_dir / src_name
|
| 508 |
+
if src_path.exists():
|
| 509 |
+
shutil.copy2(src_path, sample_dir / dst_name)
|
| 510 |
+
|
| 511 |
+
sample_meta = {
|
| 512 |
+
"run_dir": str(run_dir),
|
| 513 |
+
"ckpt_step": ckpt_step,
|
| 514 |
+
"source_motion_dir": str(motion_dir),
|
| 515 |
+
"video_path": str(video_path),
|
| 516 |
+
"instruction": instruction,
|
| 517 |
+
"output_dir": str(sample_dir),
|
| 518 |
+
"num_output_frames": int(pred_abs.shape[0]),
|
| 519 |
+
"rollout_chunks": int(rollout_chunks),
|
| 520 |
+
"textop_actions": textop_actions,
|
| 521 |
+
"decode_textop_joint_from_delta": decode_textop_joint_from_delta,
|
| 522 |
+
"num_past_frames": int(num_past_frames),
|
| 523 |
+
"enable_rtc": enable_rtc,
|
| 524 |
+
"action_exec_horizon": int(action_exec_horizon),
|
| 525 |
+
"video_sample_fps": float(video_sample_fps),
|
| 526 |
+
"video_size_pixels": int(video_size_pixels),
|
| 527 |
+
"video_pixel_cache_dtype": video_pixel_cache_dtype,
|
| 528 |
+
"video_decoder": video_decoder,
|
| 529 |
+
"video_embed_path": video_embed_path,
|
| 530 |
+
"source_video_fps": float(source_video_fps),
|
| 531 |
+
"video_duration_sec": float(video_duration_sec),
|
| 532 |
+
"full_video": not bool(args.video_clip),
|
| 533 |
+
"video_config_key": str(args.video_config_key),
|
| 534 |
+
"video_clip_motion_fps": float(video_clip_motion_fps),
|
| 535 |
+
"use_video_instruction": bool(cfg_get(video_cfg, "use_video_instruction", False)),
|
| 536 |
+
"use_audio_in_video": False,
|
| 537 |
+
}
|
| 538 |
+
(sample_dir / "infer_meta.json").write_text(json.dumps(sample_meta, ensure_ascii=False, indent=2), encoding="utf-8")
|
| 539 |
+
(output_dir / "manifest.json").write_text(
|
| 540 |
+
json.dumps({"samples": [sample_meta]}, ensure_ascii=False, indent=2),
|
| 541 |
+
encoding="utf-8",
|
| 542 |
+
)
|
| 543 |
+
print(f"[OK] video sample -> {sample_dir}")
|
| 544 |
+
print(f"[DONE] wrote manifest: {output_dir / 'manifest.json'}")
|
| 545 |
+
|
| 546 |
+
|
| 547 |
+
if __name__ == "__main__":
|
| 548 |
+
main()
|
scripts/infer/third_party_checkpoint_probe.py
ADDED
|
@@ -0,0 +1,434 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#!/usr/bin/env python3
|
| 2 |
+
"""Fixed train-only, closed-loop probes with durable prediction evidence.
|
| 3 |
+
|
| 4 |
+
Called synchronously at a source boundary by the third-party trainer. No GT
|
| 5 |
+
pose after frame zero is fed to the model; no input waveform is re-mixed.
|
| 6 |
+
"""
|
| 7 |
+
from __future__ import annotations
|
| 8 |
+
|
| 9 |
+
import argparse
|
| 10 |
+
import contextlib
|
| 11 |
+
import copy
|
| 12 |
+
import hashlib
|
| 13 |
+
import json
|
| 14 |
+
import os
|
| 15 |
+
from pathlib import Path
|
| 16 |
+
import random
|
| 17 |
+
import shutil
|
| 18 |
+
import subprocess
|
| 19 |
+
import sys
|
| 20 |
+
import time
|
| 21 |
+
import traceback
|
| 22 |
+
from typing import Any, Iterator, Sequence
|
| 23 |
+
import wave
|
| 24 |
+
|
| 25 |
+
import numpy as np
|
| 26 |
+
import torch
|
| 27 |
+
|
| 28 |
+
PROJECT_ROOT = Path(__file__).resolve().parents[2]
|
| 29 |
+
FPS = 50
|
| 30 |
+
RATE = 16000
|
| 31 |
+
PREVIEW_FPS = 25
|
| 32 |
+
PREVIEW_WIDTH = 320
|
| 33 |
+
PREVIEW_HEIGHT = 240
|
| 34 |
+
PREVIEW_CPU_THREADS = 4
|
| 35 |
+
PROFILES = ("single_short", "multi_single", "multi_babble", "mixed_short",
|
| 36 |
+
"continuous", "same_voice_aside")
|
| 37 |
+
|
| 38 |
+
|
| 39 |
+
def _sha256(path: Path) -> str:
|
| 40 |
+
digest = hashlib.sha256()
|
| 41 |
+
with path.open("rb") as handle:
|
| 42 |
+
for chunk in iter(lambda: handle.read(1024 * 1024), b""):
|
| 43 |
+
digest.update(chunk)
|
| 44 |
+
return digest.hexdigest()
|
| 45 |
+
|
| 46 |
+
|
| 47 |
+
def _write_json(path: Path, value: Any) -> None:
|
| 48 |
+
temporary = path.with_suffix(path.suffix + ".tmp")
|
| 49 |
+
temporary.write_text(json.dumps(value, indent=2, ensure_ascii=False,
|
| 50 |
+
allow_nan=False) + "\n", encoding="utf-8")
|
| 51 |
+
os.replace(temporary, path)
|
| 52 |
+
|
| 53 |
+
|
| 54 |
+
def validate_probe_recipes(recipes: Sequence[dict[str, Any]], *, probe_scope: str = "checkpoint") -> None:
|
| 55 |
+
ids = [str(row["session_id"]) for row in recipes]
|
| 56 |
+
if probe_scope not in {"checkpoint", "smoke"}:
|
| 57 |
+
raise ValueError("Unknown probe scope")
|
| 58 |
+
if any(row.get("split") != "train" for row in recipes):
|
| 59 |
+
raise ValueError("Checkpoint probes may only use the selected training sessions")
|
| 60 |
+
if probe_scope == "smoke":
|
| 61 |
+
if not 1 <= len(ids) <= 12 or len(set(ids)) != len(ids):
|
| 62 |
+
raise ValueError("Smoke probes require 1–12 distinct train sessions")
|
| 63 |
+
if any(row.get("session_type") not in PROFILES for row in recipes):
|
| 64 |
+
raise ValueError("Unknown third-party smoke profile")
|
| 65 |
+
return
|
| 66 |
+
if len(ids) != 12 or len(set(ids)) != 12:
|
| 67 |
+
raise ValueError("Checkpoint probes require 12 distinct fixed train sessions")
|
| 68 |
+
for profile in PROFILES:
|
| 69 |
+
rows = [row for row in recipes if row.get("session_type") == profile]
|
| 70 |
+
if len(rows) != 2:
|
| 71 |
+
raise ValueError(f"Checkpoint probes require exactly two sessions for {profile}")
|
| 72 |
+
languages = {row["background_assignment"]["language"] for row in rows}
|
| 73 |
+
if languages != {"zh", "en"}:
|
| 74 |
+
raise ValueError(f"Checkpoint probes need zh/en backgrounds for {profile}")
|
| 75 |
+
|
| 76 |
+
|
| 77 |
+
def load_frozen_probe(production_root: Path, recipe: dict[str, Any]) -> tuple[
|
| 78 |
+
np.ndarray, dict[str, np.ndarray], Path]:
|
| 79 |
+
"""Hash-verify and decode PCM directly; never normalize or re-sample."""
|
| 80 |
+
audio_path = (production_root / recipe["audio"]["path"]).resolve()
|
| 81 |
+
timeline_path = (production_root / recipe["timeline"]["path"]).resolve()
|
| 82 |
+
for path, expected in ((audio_path, recipe["audio"]["sha256"]),
|
| 83 |
+
(timeline_path, recipe["timeline"]["file_sha256"])):
|
| 84 |
+
if _sha256(path) != expected:
|
| 85 |
+
raise RuntimeError(f"Frozen probe input hash changed: {path}")
|
| 86 |
+
with wave.open(str(audio_path), "rb") as source:
|
| 87 |
+
if (source.getnchannels(), source.getsampwidth(), source.getframerate()) != (1, 2, RATE):
|
| 88 |
+
raise ValueError("Frozen probe audio must be mono 16 kHz PCM16")
|
| 89 |
+
if source.getnframes() != int(recipe["audio"]["sample_count"]):
|
| 90 |
+
raise ValueError("Frozen audio length no longer matches its recipe")
|
| 91 |
+
waveform = np.frombuffer(source.readframes(source.getnframes()), dtype="<i2").astype(np.float32)
|
| 92 |
+
waveform /= np.float32(32768.0)
|
| 93 |
+
with np.load(timeline_path, allow_pickle=False) as stored:
|
| 94 |
+
timeline = {key: stored[key].copy() for key in ("states", "control_label")}
|
| 95 |
+
states, labels = timeline["states"], timeline["control_label"]
|
| 96 |
+
if states.ndim != 2 or states.shape[1] != 36 or len(states) < 2:
|
| 97 |
+
raise ValueError("Invalid frozen GT states")
|
| 98 |
+
if labels.shape != (len(states),) or not np.isin(labels, (0, 1, 2)).all():
|
| 99 |
+
raise ValueError("Invalid frozen GT control labels")
|
| 100 |
+
if not np.isfinite(states).all():
|
| 101 |
+
raise ValueError("Non-finite frozen GT states")
|
| 102 |
+
return waveform, timeline, audio_path
|
| 103 |
+
|
| 104 |
+
|
| 105 |
+
def score_prediction(motion: np.ndarray, metadata: dict[str, Any],
|
| 106 |
+
timeline: dict[str, np.ndarray], recipe: dict[str, Any]) -> dict[str, Any]:
|
| 107 |
+
"""Report pose errors and control decisions separately, with explicit alignment."""
|
| 108 |
+
predicted = np.asarray(motion, dtype=np.float64)
|
| 109 |
+
states = np.asarray(timeline["states"], dtype=np.float64)
|
| 110 |
+
labels = np.asarray(timeline["control_label"], dtype=np.int64)
|
| 111 |
+
if predicted.ndim != 2 or predicted.shape[1] != 36 or not np.isfinite(predicted).all():
|
| 112 |
+
raise ValueError("Non-finite or malformed model prediction")
|
| 113 |
+
pose_count = min(len(predicted), len(states) - 1)
|
| 114 |
+
if pose_count <= 0:
|
| 115 |
+
raise ValueError("No aligned predicted/GT poses")
|
| 116 |
+
# Training actions at t predict t+1, while compact states store t.
|
| 117 |
+
delta = predicted[:pose_count] - states[1:pose_count + 1]
|
| 118 |
+
pred_quat = predicted[:pose_count, 32:36]
|
| 119 |
+
gt_quat = states[1:pose_count + 1, 32:36]
|
| 120 |
+
pred_norm = np.linalg.norm(pred_quat, axis=1)
|
| 121 |
+
gt_norm = np.linalg.norm(gt_quat, axis=1)
|
| 122 |
+
if np.any(pred_norm <= 1e-8) or np.any(gt_norm <= 1e-8):
|
| 123 |
+
raise ValueError("Zero root quaternion in predicted/GT pose")
|
| 124 |
+
quat_similarity = np.abs(np.sum(pred_quat * gt_quat, axis=1) / (pred_norm * gt_norm))
|
| 125 |
+
count = min(len(predicted), len(labels))
|
| 126 |
+
controls = np.full(len(predicted), -1, dtype=np.int64)
|
| 127 |
+
events = metadata["control_events"]
|
| 128 |
+
for index, event in enumerate(events):
|
| 129 |
+
start = int(event["output_frame"])
|
| 130 |
+
end = int(events[index + 1]["output_frame"]) if index + 1 < len(events) else len(predicted)
|
| 131 |
+
control_id = int(event["predicted_id"])
|
| 132 |
+
if not 0 <= start < end <= len(predicted) or control_id not in (0, 1, 2):
|
| 133 |
+
raise ValueError("Invalid prediction control timeline")
|
| 134 |
+
controls[start:end] = control_id
|
| 135 |
+
if np.any(controls < 0):
|
| 136 |
+
raise ValueError("Prediction control events do not cover the trajectory")
|
| 137 |
+
controls = controls[:count]
|
| 138 |
+
gt = labels[:count]
|
| 139 |
+
confusion = np.zeros((3, 3), dtype=np.int64)
|
| 140 |
+
np.add.at(confusion, (gt, controls), 1)
|
| 141 |
+
# Background-only is NOT synonymous with idle: a valid earlier command can
|
| 142 |
+
# still be executing. False activation is scored only when GT is non-motion.
|
| 143 |
+
sample_starts = np.arange(count) * (RATE // FPS)
|
| 144 |
+
background = np.zeros(count, dtype=bool)
|
| 145 |
+
target = np.zeros(count, dtype=bool)
|
| 146 |
+
for event in recipe.get("interference_events", []):
|
| 147 |
+
background |= ((sample_starts < int(event["end_sample"])) &
|
| 148 |
+
(sample_starts + RATE // FPS > int(event["start_sample"])))
|
| 149 |
+
for event in recipe.get("target_speech_events", []):
|
| 150 |
+
target |= ((sample_starts < int(event["end_sample"])) &
|
| 151 |
+
(sample_starts + RATE // FPS > int(event["start_sample"])))
|
| 152 |
+
background_nonmotion = background & ~target & (gt != 1)
|
| 153 |
+
no_motion = gt != 1
|
| 154 |
+
motion_gt = gt == 1
|
| 155 |
+
def rate(values: np.ndarray, mask: np.ndarray) -> float | None:
|
| 156 |
+
return float(values[mask].mean()) if mask.any() else None
|
| 157 |
+
return {
|
| 158 |
+
"pose_alignment": "predicted[t] compared with frozen state[t+1]; no DTW/realignment",
|
| 159 |
+
"control_alignment": "control predicted at t compared with frozen control_label[t]",
|
| 160 |
+
"pose_scored_frames": pose_count, "control_scored_frames": count,
|
| 161 |
+
"prediction_frames": len(predicted),
|
| 162 |
+
"unscored_prediction_tail_frames": len(predicted) - pose_count,
|
| 163 |
+
"joint_mae_radians": float(np.abs(delta[:, :29]).mean()),
|
| 164 |
+
"root_position_rmse_metres": float(np.sqrt(np.square(delta[:, 29:32]).sum(axis=1).mean())),
|
| 165 |
+
"root_rotation_mae_radians": float((2 * np.arccos(np.clip(quat_similarity, 0, 1))).mean()),
|
| 166 |
+
"control_accuracy": float((controls == gt).mean()),
|
| 167 |
+
"control_confusion_rows_gt_columns_prediction": confusion.tolist(),
|
| 168 |
+
"control_order": ["listen", "motion", "idle"],
|
| 169 |
+
"nonmotion_gt_frames": int(no_motion.sum()),
|
| 170 |
+
"false_motion_rate": rate(controls == 1, no_motion),
|
| 171 |
+
"motion_gt_frames": int(motion_gt.sum()),
|
| 172 |
+
"missed_motion_rate": rate(controls != 1, motion_gt),
|
| 173 |
+
"background_only_nonmotion_gt_frames": int(background_nonmotion.sum()),
|
| 174 |
+
"background_only_false_motion_rate": rate(controls == 1, background_nonmotion),
|
| 175 |
+
"root_height_min_metres": float(predicted[:, 31].min()),
|
| 176 |
+
"root_height_max_metres": float(predicted[:, 31].max()),
|
| 177 |
+
"joint_step_rms_radians": float(np.sqrt(np.square(np.diff(predicted[:, :29], axis=0)).mean())),
|
| 178 |
+
"interpretation": "Train memorization/robustness diagnostic, not held-out generalization or physics success.",
|
| 179 |
+
}
|
| 180 |
+
|
| 181 |
+
|
| 182 |
+
@contextlib.contextmanager
|
| 183 |
+
def preserve_training_state(model: Any, seed: int) -> Iterator[None]:
|
| 184 |
+
"""Probe must not alter dropout flags, RNG streams, or training scheduler."""
|
| 185 |
+
module_flags = [(module, module.training) for module in model.modules()]
|
| 186 |
+
python_rng, numpy_rng = random.getstate(), np.random.get_state()
|
| 187 |
+
torch_rng = torch.get_rng_state()
|
| 188 |
+
cuda_rng = torch.cuda.get_rng_state_all() if torch.cuda.is_initialized() else None
|
| 189 |
+
old_scheduler = getattr(model, "noise_scheduler", None)
|
| 190 |
+
try:
|
| 191 |
+
if old_scheduler is not None:
|
| 192 |
+
model.noise_scheduler = copy.deepcopy(old_scheduler)
|
| 193 |
+
random.seed(seed)
|
| 194 |
+
np.random.seed(seed % (2 ** 32))
|
| 195 |
+
torch.manual_seed(seed)
|
| 196 |
+
model.eval()
|
| 197 |
+
with torch.inference_mode():
|
| 198 |
+
yield
|
| 199 |
+
finally:
|
| 200 |
+
for module, training in module_flags:
|
| 201 |
+
module.training = training
|
| 202 |
+
if old_scheduler is not None:
|
| 203 |
+
model.noise_scheduler = old_scheduler
|
| 204 |
+
random.setstate(python_rng)
|
| 205 |
+
np.random.set_state(numpy_rng)
|
| 206 |
+
torch.set_rng_state(torch_rng)
|
| 207 |
+
if cuda_rng is not None:
|
| 208 |
+
torch.cuda.set_rng_state_all(cuda_rng)
|
| 209 |
+
|
| 210 |
+
|
| 211 |
+
@contextlib.contextmanager
|
| 212 |
+
def inference_bindings(model: Any, launch_config: Any, vlm_processor: Any) -> Iterator[None]:
|
| 213 |
+
"""Training keeps processor/scheduler on the trainer, unlike loaded inference."""
|
| 214 |
+
from diffusers.schedulers.scheduling_flow_match_euler_discrete import FlowMatchEulerDiscreteScheduler
|
| 215 |
+
|
| 216 |
+
missing = object()
|
| 217 |
+
names = ("device", "vlm_processor", "noise_scheduler")
|
| 218 |
+
previous = {name: getattr(model, name, missing) for name in names}
|
| 219 |
+
cfg = getattr(launch_config, "model", None)
|
| 220 |
+
if str(getattr(cfg, "noise_scheduler", "flow")) != "flow":
|
| 221 |
+
raise ValueError("Closed-loop third-party probes require the flow scheduler")
|
| 222 |
+
processor = vlm_processor if vlm_processor is not None else getattr(model, "vlm_processor", None)
|
| 223 |
+
if processor is None:
|
| 224 |
+
raise ValueError("In-process checkpoint probe requires trainer.vlm_processor")
|
| 225 |
+
try:
|
| 226 |
+
model.device = next(model.parameters()).device
|
| 227 |
+
model.vlm_processor = processor
|
| 228 |
+
model.noise_scheduler = FlowMatchEulerDiscreteScheduler(
|
| 229 |
+
num_train_timesteps=int(getattr(cfg, "train_diffusion_steps", 1000)))
|
| 230 |
+
yield
|
| 231 |
+
finally:
|
| 232 |
+
for name, value in previous.items():
|
| 233 |
+
if value is missing:
|
| 234 |
+
if hasattr(model, name):
|
| 235 |
+
delattr(model, name)
|
| 236 |
+
else:
|
| 237 |
+
setattr(model, name, value)
|
| 238 |
+
|
| 239 |
+
|
| 240 |
+
def _run(command: Sequence[str], log: Path, **kwargs: Any) -> None:
|
| 241 |
+
with log.open("a", encoding="utf-8") as handle:
|
| 242 |
+
result = subprocess.run(list(command), stdout=handle, stderr=subprocess.STDOUT,
|
| 243 |
+
check=False, timeout=1800, **kwargs)
|
| 244 |
+
if result.returncode:
|
| 245 |
+
raise RuntimeError(f"Probe render failed (exit {result.returncode}); see {log}")
|
| 246 |
+
|
| 247 |
+
|
| 248 |
+
def render_prediction(sample_dir: Path, audio_path: Path) -> Path:
|
| 249 |
+
"""Low-cost CPU review video; the underlying predicted CSV remains 50 Hz."""
|
| 250 |
+
from scripts.data.render_hfd_third_party_gt import (
|
| 251 |
+
GMR_PYTHON, GMR_XML, RENDER_ROOT, _render_environment,
|
| 252 |
+
)
|
| 253 |
+
from scripts.infer.mux_full_duplex_audio import probe as probe_media, stream
|
| 254 |
+
from scripts.third_party_temp import project_temp_directory
|
| 255 |
+
|
| 256 |
+
environment = _render_environment("cpu", cpu_threads=PREVIEW_CPU_THREADS)
|
| 257 |
+
scratch = str(project_temp_directory(PROJECT_ROOT))
|
| 258 |
+
environment.update({name: scratch for name in ("TMPDIR", "TMP", "TEMP", "SONIC_RENDER_TMPDIR")})
|
| 259 |
+
meta_path = sample_dir / "infer_meta.json"
|
| 260 |
+
metadata = json.loads(meta_path.read_text(encoding="utf-8"))
|
| 261 |
+
metadata["review_render"] = {
|
| 262 |
+
"motion_fps": FPS, "video_fps": PREVIEW_FPS,
|
| 263 |
+
"width": PREVIEW_WIDTH, "height": PREVIEW_HEIGHT,
|
| 264 |
+
"cpu_threads": PREVIEW_CPU_THREADS,
|
| 265 |
+
"purpose": "Checkpoint screening preview; full 50Hz predicted poses and metrics retained",
|
| 266 |
+
"audio_policy": "Complete original mixed WAV, no time stretch or offset",
|
| 267 |
+
}
|
| 268 |
+
_write_json(meta_path, metadata)
|
| 269 |
+
log = sample_dir / "render.log"
|
| 270 |
+
_run([str(GMR_PYTHON), str(RENDER_ROOT / "scripts/batch_render_reference_motion_mujoco.py"),
|
| 271 |
+
"--motion-dir", str(sample_dir), "--xml-path", str(GMR_XML),
|
| 272 |
+
"--source-fps", str(FPS), "--target-fps", str(PREVIEW_FPS),
|
| 273 |
+
"--width", str(PREVIEW_WIDTH), "--height", str(PREVIEW_HEIGHT),
|
| 274 |
+
"--dof-order", "isaaclab", "--camera-distance", "2.5"], log,
|
| 275 |
+
cwd=sample_dir, env=environment)
|
| 276 |
+
# The shared renderer calls every CSV input 'reference'; rename it here so
|
| 277 |
+
# users do not confuse the actual model prediction with the dataset's GT.
|
| 278 |
+
silent = sample_dir / "prediction_silent.mp4"
|
| 279 |
+
(sample_dir / "reference_motion.mp4").replace(silent)
|
| 280 |
+
media = probe_media(silent)
|
| 281 |
+
video_stream = stream(media, "video")
|
| 282 |
+
with wave.open(str(audio_path), "rb") as source:
|
| 283 |
+
audio_seconds = source.getnframes() / source.getframerate()
|
| 284 |
+
video_seconds = float(video_stream.get("duration") or media["format"]["duration"])
|
| 285 |
+
if (video_stream["width"], video_stream["height"]) != (PREVIEW_WIDTH, PREVIEW_HEIGHT):
|
| 286 |
+
raise RuntimeError("Checkpoint preview resolution drifted")
|
| 287 |
+
if video_seconds < audio_seconds - 1e-6 or video_seconds - audio_seconds > 1 / PREVIEW_FPS + 1e-6:
|
| 288 |
+
raise RuntimeError("Checkpoint preview would truncate or extend the original audio timeline")
|
| 289 |
+
video = sample_dir / "prediction_with_audio.mp4"
|
| 290 |
+
_run([sys.executable, str(PROJECT_ROOT / "scripts/infer/mux_full_duplex_audio.py"),
|
| 291 |
+
"--video", str(silent), "--audio", str(audio_path), "--output", str(video),
|
| 292 |
+
"--infer-meta", str(sample_dir / "infer_meta.json"), "--audio-is-timeline",
|
| 293 |
+
"--expected-fps", str(PREVIEW_FPS)], log, env=environment)
|
| 294 |
+
_run(["ffmpeg", "-v", "error", "-xerror", "-i", str(video),
|
| 295 |
+
"-f", "null", "-"], log, env=environment)
|
| 296 |
+
return video
|
| 297 |
+
|
| 298 |
+
|
| 299 |
+
def _aggregate(rows: Sequence[dict[str, Any]]) -> dict[str, Any]:
|
| 300 |
+
keys = ("joint_mae_radians", "root_position_rmse_metres", "control_accuracy",
|
| 301 |
+
"root_rotation_mae_radians",
|
| 302 |
+
"false_motion_rate", "missed_motion_rate", "background_only_false_motion_rate")
|
| 303 |
+
result: dict[str, Any] = {"averaging": "unweighted mean over available session metrics"}
|
| 304 |
+
for key in keys:
|
| 305 |
+
values = [row["metrics"][key] for row in rows if row["metrics"].get(key) is not None]
|
| 306 |
+
result[key] = float(np.mean(values)) if values else None
|
| 307 |
+
result[key + "_sessions"] = len(values)
|
| 308 |
+
return result
|
| 309 |
+
|
| 310 |
+
|
| 311 |
+
def run_checkpoint_probes(model: Any, launch_config: Any, *, production_root: str | Path,
|
| 312 |
+
recipes: Sequence[dict[str, Any]], output_root: str | Path,
|
| 313 |
+
global_step: int, seed: int = 260930,
|
| 314 |
+
num_inference_steps: int = 10,
|
| 315 |
+
probe_scope: str = "checkpoint",
|
| 316 |
+
vlm_processor: Any = None) -> dict[str, Any]:
|
| 317 |
+
"""Run all fixed 12 predictions; persist partial failure and re-raise errors.
|
| 318 |
+
|
| 319 |
+
The trainer must call only at a source boundary, and pass recipes from its
|
| 320 |
+
frozen 300-session selection. output_root is normally run/checkpoint_probes.
|
| 321 |
+
"""
|
| 322 |
+
from scripts.infer import full_duplex_audio_to_motion as infer
|
| 323 |
+
|
| 324 |
+
validate_probe_recipes(recipes, probe_scope=probe_scope)
|
| 325 |
+
if getattr(model, "control_condition_inference_mode", "hard") != "hard":
|
| 326 |
+
raise ValueError("Third-party checkpoint probes require own hard control conditioning")
|
| 327 |
+
if int(global_step) < 0 or int(num_inference_steps) <= 0:
|
| 328 |
+
raise ValueError("Invalid probe step/inference steps")
|
| 329 |
+
production_root = Path(production_root).resolve()
|
| 330 |
+
output_root = Path(output_root).resolve()
|
| 331 |
+
if not output_root.is_relative_to(PROJECT_ROOT.resolve()):
|
| 332 |
+
raise ValueError("Probe outputs must remain inside HFD_lzk")
|
| 333 |
+
step_dir = output_root / f"step_{int(global_step):08d}"
|
| 334 |
+
step_dir.mkdir(parents=True, exist_ok=False)
|
| 335 |
+
started = time.monotonic()
|
| 336 |
+
report: dict[str, Any] = {
|
| 337 |
+
"format": "hfd_third_party_checkpoint_probe_v1", "status": "running",
|
| 338 |
+
"global_step": int(global_step), "seed": int(seed), "evaluation_split": "train_probe",
|
| 339 |
+
"probe_scope": probe_scope,
|
| 340 |
+
"fixed_session_ids": [row["session_id"] for row in recipes],
|
| 341 |
+
"input_policy": "Exact frozen mixed PCM16; no remix, resampling, gain change, or video input",
|
| 342 |
+
"rollout_policy": "Own hard control and predicted state/RTC; only initial GT pose supplied",
|
| 343 |
+
"tail_policy": "Final 0.2s uses last predicted chunk tail, not a GT or stationary hold",
|
| 344 |
+
"render_backend": "mesa_llvmpipe_cpu", "motion_fps": FPS,
|
| 345 |
+
"review_video_fps": PREVIEW_FPS, "review_video_resolution": [PREVIEW_WIDTH, PREVIEW_HEIGHT],
|
| 346 |
+
"render_cpu_threads": PREVIEW_CPU_THREADS, "samples": [],
|
| 347 |
+
}
|
| 348 |
+
_write_json(step_dir / "report.json", report)
|
| 349 |
+
try:
|
| 350 |
+
with preserve_training_state(model, seed), inference_bindings(model, launch_config, vlm_processor):
|
| 351 |
+
for index, recipe in enumerate(recipes, 1):
|
| 352 |
+
session_id = str(recipe["session_id"])
|
| 353 |
+
sample_dir = step_dir / f"{index:02d}_{recipe['session_type']}_{session_id[:12]}"
|
| 354 |
+
sample_dir.mkdir()
|
| 355 |
+
waveform, timeline, audio_path = load_frozen_probe(production_root, recipe)
|
| 356 |
+
print(f"[train_probe] step={global_step} sample={index}/{len(recipes)} session={session_id}", flush=True)
|
| 357 |
+
motion, metadata = infer.rollout_full_duplex(
|
| 358 |
+
model=model, background_waveforms=[], waveform=waveform,
|
| 359 |
+
prompt=infer.DEFAULT_COMMAND_PROMPT, source_id=f"train_probe:{session_id}",
|
| 360 |
+
init_abs=timeline["states"][0].copy(), launch_config=launch_config,
|
| 361 |
+
motion_frames=max(int(getattr(launch_config.data, "turn_transition_frames", 50)), 50) + 300,
|
| 362 |
+
num_inference_steps=int(num_inference_steps), respect_control_token=True,
|
| 363 |
+
max_listen_tail_steps=1, initial_idle_steps=0,
|
| 364 |
+
seed=seed + index, frozen_audio_timeline=True,
|
| 365 |
+
)
|
| 366 |
+
expected = int(round(len(waveform) * FPS / RATE))
|
| 367 |
+
if motion.shape != (expected, 36):
|
| 368 |
+
raise RuntimeError("Frozen probe prediction does not cover its exact audio timeline")
|
| 369 |
+
metrics = score_prediction(motion, metadata, timeline, recipe)
|
| 370 |
+
metadata.update({"session_id": session_id, "global_step": int(global_step),
|
| 371 |
+
"frozen_audio_path": str(audio_path),
|
| 372 |
+
"frozen_audio_sha256": recipe["audio"]["sha256"],
|
| 373 |
+
"artifact_role": "model_prediction_not_GT"})
|
| 374 |
+
infer.export_motion(sample_dir, motion)
|
| 375 |
+
_write_json(sample_dir / "infer_meta.json", metadata)
|
| 376 |
+
_write_json(sample_dir / "metrics.json", metrics)
|
| 377 |
+
_write_json(sample_dir / "session_recipe.json", recipe)
|
| 378 |
+
video = render_prediction(sample_dir, audio_path)
|
| 379 |
+
row = {"session_id": session_id, "profile": recipe["session_type"],
|
| 380 |
+
"background_language": recipe["background_assignment"]["language"],
|
| 381 |
+
"background_sir_db": recipe.get("mix_metadata", {}).get("measured_background_sir_db"),
|
| 382 |
+
"metrics": metrics, "video": str(video), "video_sha256": _sha256(video),
|
| 383 |
+
"motion_sha256": _sha256(sample_dir / "motion.npy"),
|
| 384 |
+
"full_video_decode_passed": True}
|
| 385 |
+
report["samples"].append(row)
|
| 386 |
+
_write_json(step_dir / "report.json", report)
|
| 387 |
+
report["status"] = "complete"
|
| 388 |
+
report["aggregate"] = _aggregate(report["samples"])
|
| 389 |
+
except BaseException as error:
|
| 390 |
+
report["status"] = "failed"
|
| 391 |
+
report["error"] = f"{type(error).__name__}: {error}"
|
| 392 |
+
(step_dir / "failure.txt").write_text(traceback.format_exc(), encoding="utf-8")
|
| 393 |
+
raise
|
| 394 |
+
finally:
|
| 395 |
+
report["elapsed_seconds"] = time.monotonic() - started
|
| 396 |
+
_write_json(step_dir / "report.json", report)
|
| 397 |
+
_write_json(output_root / "latest.json", {"report": str(step_dir / "report.json"),
|
| 398 |
+
"status": report["status"], "global_step": global_step})
|
| 399 |
+
with (output_root / "history.jsonl").open("a", encoding="utf-8") as handle:
|
| 400 |
+
handle.write(json.dumps({"global_step": global_step, "status": report["status"],
|
| 401 |
+
"report": str(step_dir / "report.json"),
|
| 402 |
+
"elapsed_seconds": report["elapsed_seconds"],
|
| 403 |
+
"aggregate": report.get("aggregate", {})}, ensure_ascii=False) + "\n")
|
| 404 |
+
lines = ["# Checkpoint 训练集闭环记录", "", f"step={global_step},状态:{report['status']}。",
|
| 405 |
+
"这些是模型预测,不是 GT;训练集结果不代表泛化能力。", "",
|
| 406 |
+
"审核视频为 320×240 / 25 fps;完整预测轨迹和指标保留 50 Hz,原始混音时间线不变。", "",
|
| 407 |
+
"| 序号 | 类型 | 背景语言 | 控制准确率 | 视频 |", "|---|---|---|---:|---|"]
|
| 408 |
+
for index, row in enumerate(report["samples"], 1):
|
| 409 |
+
relative_video = Path(row["video"]).relative_to(step_dir)
|
| 410 |
+
lines.append(f"| {index:02d} | {row['profile']} | {row['background_language']} | "
|
| 411 |
+
f"{row['metrics']['control_accuracy']:.3f} | [模型预测]({relative_video}) |")
|
| 412 |
+
if report.get("error"):
|
| 413 |
+
lines.extend(["", "失败原因见 failure.txt;未生成的样本不得视为通过。"])
|
| 414 |
+
(step_dir / "README.md").write_text("\n".join(lines) + "\n", encoding="utf-8")
|
| 415 |
+
return report
|
| 416 |
+
|
| 417 |
+
|
| 418 |
+
def check_renderer() -> dict[str, Any]:
|
| 419 |
+
from scripts.data.render_hfd_third_party_gt import GMR_PYTHON, GMR_XML, RENDER_ROOT
|
| 420 |
+
required = (GMR_PYTHON, GMR_XML, RENDER_ROOT / "scripts/batch_render_reference_motion_mujoco.py")
|
| 421 |
+
missing = [str(path) for path in required if not path.is_file()]
|
| 422 |
+
missing += [name for name in ("ffmpeg", "ffprobe") if not shutil.which(name)]
|
| 423 |
+
if missing:
|
| 424 |
+
raise FileNotFoundError(f"Missing checkpoint video renderer requirements: {missing}")
|
| 425 |
+
return {"paths_present": True, "backend": "cpu", "render_execution_verified": False}
|
| 426 |
+
|
| 427 |
+
|
| 428 |
+
if __name__ == "__main__":
|
| 429 |
+
if str(PROJECT_ROOT) not in sys.path:
|
| 430 |
+
sys.path.insert(0, str(PROJECT_ROOT))
|
| 431 |
+
parser = argparse.ArgumentParser(description=__doc__)
|
| 432 |
+
parser.add_argument("--check-renderer", action="store_true", required=True)
|
| 433 |
+
parser.parse_args()
|
| 434 |
+
print(json.dumps(check_renderer(), ensure_ascii=False))
|
scripts/infer/third_party_scale_metrics.py
ADDED
|
@@ -0,0 +1,172 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#!/usr/bin/env python3
|
| 2 |
+
"""GPU-0 scale-stage closed-loop metrics on separately frozen train/validation."""
|
| 3 |
+
from __future__ import annotations
|
| 4 |
+
|
| 5 |
+
import argparse
|
| 6 |
+
import hashlib
|
| 7 |
+
import json
|
| 8 |
+
import os
|
| 9 |
+
from pathlib import Path
|
| 10 |
+
import subprocess
|
| 11 |
+
import sys
|
| 12 |
+
import time
|
| 13 |
+
|
| 14 |
+
PROJECT = Path(__file__).resolve().parents[2]
|
| 15 |
+
sys.path[:0] = [str(PROJECT), str(PROJECT / "src")]
|
| 16 |
+
from scripts.third_party_strategy_plan import write_json
|
| 17 |
+
from scripts.infer.third_party_metrics import aggregate, local
|
| 18 |
+
from scripts.third_party_scale_plan import SEED, safe_relative_path
|
| 19 |
+
|
| 20 |
+
|
| 21 |
+
def read(path):
|
| 22 |
+
return json.loads(Path(path).read_text())
|
| 23 |
+
|
| 24 |
+
|
| 25 |
+
def sample_seed(sid):
|
| 26 |
+
return int(hashlib.sha256(f"{SEED}:rollout:{sid}".encode()).hexdigest()[:8], 16)
|
| 27 |
+
|
| 28 |
+
|
| 29 |
+
def recipes_for(manifest, ids):
|
| 30 |
+
from psi.data.seed.third_party_frozen_dataset import read_jsonl, sha256_file
|
| 31 |
+
selection = read(manifest)
|
| 32 |
+
split = selection.get("split")
|
| 33 |
+
if (selection.get("scaleup") is not True or split not in ("train", "validation")
|
| 34 |
+
or len(set(ids)) != len(ids) or not set(ids) <= set(selection["session_ids"])):
|
| 35 |
+
raise ValueError("Invalid scale evaluation split/IDs")
|
| 36 |
+
root = local(selection["production_root"])
|
| 37 |
+
path = root / "manifests/session_recipes.jsonl"
|
| 38 |
+
if sha256_file(path) != selection["recipe_manifest_sha256"]:
|
| 39 |
+
raise ValueError("Evaluation recipe manifest changed")
|
| 40 |
+
wanted, found = set(ids), {}
|
| 41 |
+
for row in read_jsonl(path):
|
| 42 |
+
if row["session_id"] in wanted:
|
| 43 |
+
if row["session_id"] in found or row["split"] != split:
|
| 44 |
+
raise ValueError("Evaluation crossed production split")
|
| 45 |
+
for name in ("audio", "timeline"):
|
| 46 |
+
safe_relative_path(root, row[name]["path"])
|
| 47 |
+
found[row["session_id"]] = row
|
| 48 |
+
if set(found) != wanted:
|
| 49 |
+
raise ValueError("Missing evaluation recipe")
|
| 50 |
+
return root, found, split
|
| 51 |
+
|
| 52 |
+
|
| 53 |
+
def evaluate(source_run, step, manifest, output, *, full=False, workers=4, include_videos=False):
|
| 54 |
+
if os.environ.get("CUDA_VISIBLE_DEVICES") != "0" or workers not in (1, 2, 4):
|
| 55 |
+
raise RuntimeError("Use the managed single-GPU-0 scale launcher")
|
| 56 |
+
source_run, manifest, output = map(local, (source_run, manifest, output))
|
| 57 |
+
if output.exists() or not output.is_relative_to(PROJECT / "runs"):
|
| 58 |
+
raise ValueError("Fresh local evaluation directory required")
|
| 59 |
+
selection = read(manifest)
|
| 60 |
+
ids = list(selection["session_ids"] if full else selection["evaluation_session_ids"])
|
| 61 |
+
if include_videos:
|
| 62 |
+
ids.extend(sid for sid in selection["video_session_ids"] if sid not in ids)
|
| 63 |
+
if not ids or len(set(ids)) != len(ids):
|
| 64 |
+
raise ValueError("Invalid fixed evaluation panel")
|
| 65 |
+
output.mkdir(parents=True)
|
| 66 |
+
(output / "samples").mkdir()
|
| 67 |
+
task = {"source_run": str(source_run), "step": step, "manifest": str(manifest),
|
| 68 |
+
"manifest_sha256": hashlib.sha256(manifest.read_bytes()).hexdigest(),
|
| 69 |
+
"output": str(output), "split": selection["split"], "all_ids": ids}
|
| 70 |
+
write_json(output / "identity.json", task)
|
| 71 |
+
started = time.monotonic()
|
| 72 |
+
children, logs = [], []
|
| 73 |
+
try:
|
| 74 |
+
for rank in range(min(workers, len(ids))):
|
| 75 |
+
path = output / f"worker_{rank}.json"
|
| 76 |
+
write_json(path, {**task, "ids": ids[rank::workers]})
|
| 77 |
+
log = (output / f"worker_{rank}.log").open("x"); logs.append(log)
|
| 78 |
+
children.append(subprocess.Popen([sys.executable, str(Path(__file__).resolve()), "--worker", str(path)],
|
| 79 |
+
cwd=PROJECT, stdout=log, stderr=subprocess.STDOUT))
|
| 80 |
+
while any(child.poll() is None for child in children):
|
| 81 |
+
if any(child.poll() not in (None, 0) for child in children):
|
| 82 |
+
raise RuntimeError(f"Scale evaluation worker failed: {output}")
|
| 83 |
+
time.sleep(1)
|
| 84 |
+
if any(child.returncode != 0 for child in children):
|
| 85 |
+
raise RuntimeError(f"Scale evaluation failed: {output}")
|
| 86 |
+
finally:
|
| 87 |
+
for child in children:
|
| 88 |
+
if child.poll() is None:
|
| 89 |
+
child.terminate()
|
| 90 |
+
for child in children:
|
| 91 |
+
try:
|
| 92 |
+
child.wait(timeout=30)
|
| 93 |
+
except subprocess.TimeoutExpired:
|
| 94 |
+
child.kill(); child.wait()
|
| 95 |
+
for log in logs:
|
| 96 |
+
log.close()
|
| 97 |
+
rows = [read(output / "samples" / sid / "result.json") for sid in ids]
|
| 98 |
+
by_id = {r["session_id"]: r for r in selection["sessions"]}
|
| 99 |
+
groups = {}
|
| 100 |
+
for field in ("session_type", "background_language", "background_high"):
|
| 101 |
+
groups[field] = {value: {"count": len(subset), **aggregate(subset)}
|
| 102 |
+
for value in sorted({str(by_id[r["session_id"]][field]) for r in rows})
|
| 103 |
+
if (subset := [r for r in rows if str(by_id[r["session_id"]][field]) == value])}
|
| 104 |
+
report = {"status": "complete", "step": step, "evaluation_split": selection["split"],
|
| 105 |
+
"panel": "all_eligible" if full else "fixed_class_language_panel",
|
| 106 |
+
"count": len(rows), "samples": rows, "aggregate": aggregate(rows), "groups": groups,
|
| 107 |
+
"elapsed_seconds": time.monotonic() - started, "workers": workers, "rendered_videos": 0,
|
| 108 |
+
"input_policy": "original full mixed WAV; no GT after initial pose; no optimizer updates",
|
| 109 |
+
"seed_policy": "stable per session, independent of workers/checkpoint/panel size"}
|
| 110 |
+
panel = set(selection["evaluation_session_ids"])
|
| 111 |
+
fixed = [r for r in rows if r["session_id"] in panel]
|
| 112 |
+
report["fixed_panel_comparison"] = {"count": len(fixed), "aggregate": aggregate(fixed)}
|
| 113 |
+
write_json(output / "report.json", report)
|
| 114 |
+
return report
|
| 115 |
+
|
| 116 |
+
|
| 117 |
+
def worker(path):
|
| 118 |
+
import numpy as np
|
| 119 |
+
import torch
|
| 120 |
+
from scripts.third_party_temp import configure_project_temp
|
| 121 |
+
from scripts.infer import full_duplex_audio_to_motion as infer
|
| 122 |
+
from scripts.infer.third_party_checkpoint_probe import load_frozen_probe, score_prediction, _sha256
|
| 123 |
+
from psi.models.full_duplex_psi0 import FullDuplexPsi0Model
|
| 124 |
+
from psi.config.train.posttrain_third_party_config import DynamicLaunchConfig
|
| 125 |
+
if os.environ.get("CUDA_VISIBLE_DEVICES") != "0":
|
| 126 |
+
raise RuntimeError("Scale metrics require GPU0 isolation")
|
| 127 |
+
configure_project_temp(PROJECT); torch.set_num_threads(2)
|
| 128 |
+
task = read(local(path)); manifest = local(task["manifest"])
|
| 129 |
+
if hashlib.sha256(manifest.read_bytes()).hexdigest() != task["manifest_sha256"]:
|
| 130 |
+
raise ValueError("Metric task selection changed")
|
| 131 |
+
source = local(task["source_run"])
|
| 132 |
+
cfg = DynamicLaunchConfig.model_validate_json((source / "run_config.json").read_text())
|
| 133 |
+
infer.validate_contract(cfg)
|
| 134 |
+
model = FullDuplexPsi0Model.from_streaming_checkpoint(source, task["step"], cfg, torch.device("cuda:0"))
|
| 135 |
+
model.eval()
|
| 136 |
+
root, recipes, split = recipes_for(manifest, task["ids"])
|
| 137 |
+
if split != task["split"]:
|
| 138 |
+
raise ValueError("Metric task split changed")
|
| 139 |
+
with torch.inference_mode():
|
| 140 |
+
for sid in task["ids"]:
|
| 141 |
+
started = time.monotonic(); recipe = recipes[sid]
|
| 142 |
+
dest = local(task["output"]) / "samples" / sid; dest.mkdir()
|
| 143 |
+
waveform, timeline, audio = load_frozen_probe(root, recipe)
|
| 144 |
+
seed = sample_seed(sid)
|
| 145 |
+
motion, metadata = infer.rollout_full_duplex(model=model, background_waveforms=[], waveform=waveform,
|
| 146 |
+
prompt=infer.DEFAULT_COMMAND_PROMPT, source_id=f"scale_{split}:{sid}",
|
| 147 |
+
init_abs=timeline["states"][0].copy(), launch_config=cfg,
|
| 148 |
+
motion_frames=max(int(cfg.data.turn_transition_frames), 50) + 300,
|
| 149 |
+
num_inference_steps=10, respect_control_token=True, max_listen_tail_steps=1,
|
| 150 |
+
initial_idle_steps=0, seed=seed, frozen_audio_timeline=True)
|
| 151 |
+
if motion.shape != (round(len(waveform) * 50 / 16000), 36) or not np.isfinite(motion).all():
|
| 152 |
+
raise ValueError("Incomplete/nonfinite scale prediction")
|
| 153 |
+
metrics = score_prediction(motion, metadata, timeline, recipe)
|
| 154 |
+
metrics["interpretation"] = ("Held-out production validation; not physics success" if split == "validation"
|
| 155 |
+
else "Training fit diagnostic, not held-out generalization")
|
| 156 |
+
metadata.update(session_id=sid, global_step=task["step"], evaluation_split=split,
|
| 157 |
+
frozen_audio_path=str(audio), frozen_audio_sha256=recipe["audio"]["sha256"],
|
| 158 |
+
artifact_role="model_prediction_not_GT")
|
| 159 |
+
infer.export_motion(dest, motion)
|
| 160 |
+
for name, value in (("infer_meta", metadata), ("metrics", metrics), ("session_recipe", recipe)):
|
| 161 |
+
write_json(dest / f"{name}.json", value)
|
| 162 |
+
row = {"session_id": sid, "profile": recipe["session_type"], "split": split,
|
| 163 |
+
"background_language": recipe["background_assignment"]["language"], "seed": seed,
|
| 164 |
+
"metrics": metrics, "prediction_directory": str(dest), "motion_sha256": _sha256(dest / "motion.npy"),
|
| 165 |
+
"render_status": "not_requested", "inference_seconds": time.monotonic() - started}
|
| 166 |
+
write_json(dest / "result.json", row)
|
| 167 |
+
print(json.dumps({"session": sid, "split": split, "seconds": row["inference_seconds"]}), flush=True)
|
| 168 |
+
|
| 169 |
+
|
| 170 |
+
if __name__ == "__main__":
|
| 171 |
+
parser = argparse.ArgumentParser(); parser.add_argument("--worker", required=True)
|
| 172 |
+
worker(parser.parse_args().worker)
|
src/InternVLA-M1/InternVLA/__init__.py
ADDED
|
File without changes
|
src/InternVLA-M1/InternVLA/config/deepseeds/deepspeed_zero2.yaml
ADDED
|
@@ -0,0 +1,10 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
compute_environment: LOCAL_MACHINE
|
| 2 |
+
debug: false
|
| 3 |
+
deepspeed_config:
|
| 4 |
+
deepspeed_config_file: "InternVLA/config/deepseeds/ds_config.yaml"
|
| 5 |
+
deepspeed_multinode_launcher: standard
|
| 6 |
+
zero3_init_flag: false
|
| 7 |
+
distributed_type: DEEPSPEED
|
| 8 |
+
# main_process_port: 12345
|
| 9 |
+
num_machines: 1
|
| 10 |
+
num_processes: 8
|
src/InternVLA-M1/InternVLA/config/deepseeds/ds_config.yaml
ADDED
|
@@ -0,0 +1,28 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"fp16": {
|
| 3 |
+
"enabled": "auto",
|
| 4 |
+
"loss_scale": 0,
|
| 5 |
+
"loss_scale_window": 1000,
|
| 6 |
+
"initial_scale_power": 16,
|
| 7 |
+
"hysteresis": 2,
|
| 8 |
+
"min_loss_scale": 1
|
| 9 |
+
},
|
| 10 |
+
"bf16": {
|
| 11 |
+
"enabled": "auto"
|
| 12 |
+
},
|
| 13 |
+
"train_micro_batch_size_per_gpu": "auto",
|
| 14 |
+
"train_batch_size": "auto",
|
| 15 |
+
"gradient_accumulation_steps": 1,
|
| 16 |
+
"zero_optimization": {
|
| 17 |
+
"stage": 2,
|
| 18 |
+
"allgather_partitions": true,
|
| 19 |
+
"allgather_bucket_size": 5e8,
|
| 20 |
+
"reduce_scatter": true,
|
| 21 |
+
"reduce_bucket_size": 5e8,
|
| 22 |
+
"overlap_comm": true,
|
| 23 |
+
"contiguous_gradients": true,
|
| 24 |
+
"cpu_offload": false
|
| 25 |
+
},
|
| 26 |
+
"gradient_clipping": 1.0,
|
| 27 |
+
"steps_per_print": 10
|
| 28 |
+
}
|
src/InternVLA-M1/InternVLA/config/deepseeds/zero2.yaml
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
compute_environment: LOCAL_MACHINE
|
| 2 |
+
debug: false
|
| 3 |
+
deepspeed_config:
|
| 4 |
+
deepspeed_multinode_launcher: standard
|
| 5 |
+
offload_optimizer_device: none
|
| 6 |
+
offload_param_device: none
|
| 7 |
+
zero3_init_flag: false
|
| 8 |
+
zero_stage: 2
|
| 9 |
+
distributed_type: DEEPSPEED
|
| 10 |
+
downcast_bf16: 'no'
|
| 11 |
+
machine_rank: 0
|
| 12 |
+
main_training_function: main
|
| 13 |
+
mixed_precision: bf16
|
| 14 |
+
num_machines: 1
|
| 15 |
+
num_processes: 8
|
| 16 |
+
rdzv_backend: static
|
| 17 |
+
same_network: true
|
| 18 |
+
tpu_env: []
|
| 19 |
+
tpu_use_cluster: false
|
| 20 |
+
tpu_use_sudo: false
|
| 21 |
+
use_cpu: false
|
src/InternVLA-M1/InternVLA/config/deepseeds/zero3.json
ADDED
|
@@ -0,0 +1,28 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"fp16": {
|
| 3 |
+
"enabled": "auto",
|
| 4 |
+
"loss_scale": 0,
|
| 5 |
+
"loss_scale_window": 1000,
|
| 6 |
+
"initial_scale_power": 16,
|
| 7 |
+
"hysteresis": 2,
|
| 8 |
+
"min_loss_scale": 1
|
| 9 |
+
},
|
| 10 |
+
"bf16": {
|
| 11 |
+
"enabled": "auto"
|
| 12 |
+
},
|
| 13 |
+
"train_micro_batch_size_per_gpu": "auto",
|
| 14 |
+
"train_batch_size": "auto",
|
| 15 |
+
"gradient_accumulation_steps": "auto",
|
| 16 |
+
"zero_optimization": {
|
| 17 |
+
"stage": 3,
|
| 18 |
+
"overlap_comm": true,
|
| 19 |
+
"contiguous_gradients": true,
|
| 20 |
+
"sub_group_size": 1e9,
|
| 21 |
+
"reduce_bucket_size": "auto",
|
| 22 |
+
"stage3_prefetch_bucket_size": "auto",
|
| 23 |
+
"stage3_param_persistence_threshold": "auto",
|
| 24 |
+
"stage3_max_live_parameters": 1e9,
|
| 25 |
+
"stage3_max_reuse_distance": 1e9,
|
| 26 |
+
"stage3_gather_16bit_weights_on_model_save": true
|
| 27 |
+
}
|
| 28 |
+
}
|
src/InternVLA-M1/InternVLA/config/training/internvla_cotrain_custom.yaml
ADDED
|
@@ -0,0 +1,126 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
run_id: internvla_cotrain_custom
|
| 2 |
+
run_root_dir: results/Checkpoints
|
| 3 |
+
seed: 42
|
| 4 |
+
trackers: [jsonl, wandb]
|
| 5 |
+
wandb_entity: WANDB_ENTITY
|
| 6 |
+
wandb_project: WANDB_PROJECT
|
| 7 |
+
is_debug: false
|
| 8 |
+
|
| 9 |
+
framework:
|
| 10 |
+
framework_py: InternVLA-M1
|
| 11 |
+
qwenvl: # TODO later should not use qwenvl should use vlm directly
|
| 12 |
+
base_vlm: ./playground/Pretrained_models/Qwen2.5-VL-3B-Instruct
|
| 13 |
+
attn_implementation: flash_attention_2
|
| 14 |
+
vl_hidden_dim: 2048 # TODO should read from internal config
|
| 15 |
+
dino:
|
| 16 |
+
dino_backbone: dinov2_vits14
|
| 17 |
+
layer_qformer:
|
| 18 |
+
qformer_end_layer: 37
|
| 19 |
+
qformer_start_layer: 36
|
| 20 |
+
num_query_tokens: 64
|
| 21 |
+
input_dim: 2048 # here is VLM output dimension
|
| 22 |
+
ouptput_dim: 768 # here is action input dimension # here has a rule that ouptput_dim = action_hidden_dim; --> top2down logic should not default match, should be modified manually
|
| 23 |
+
grad_scale: 0.5 # let gradient pass through this module to decay, avoid destroying VLM, # seems to affect learning
|
| 24 |
+
|
| 25 |
+
action_model:
|
| 26 |
+
action_model_type: DiT-B
|
| 27 |
+
action_hidden_dim: 768
|
| 28 |
+
action_dim: 7
|
| 29 |
+
use_ema: false
|
| 30 |
+
future_action_window_size: 15
|
| 31 |
+
past_action_window_size: 0
|
| 32 |
+
repeated_diffusion_steps: 8
|
| 33 |
+
|
| 34 |
+
fm_head_config:
|
| 35 |
+
input_embedding_dim: 1536 # Action embedding after dimension 48 * 32,and DiT's attention_head_dim * attention_heads corresponds to ActionEncoder
|
| 36 |
+
hidden_size: 1024 # and DiT's last projection corresponds to ActionDecoder
|
| 37 |
+
add_pos_embed: True
|
| 38 |
+
max_seq_len: 1024
|
| 39 |
+
action_dim: 7
|
| 40 |
+
future_action_window_size: 15
|
| 41 |
+
action_horizon: 16 # = future_action_window_size + 1 + past_action_window_size
|
| 42 |
+
past_action_window_size: 0
|
| 43 |
+
noise_beta_alpha: 1.5
|
| 44 |
+
noise_beta_beta: 1.0
|
| 45 |
+
noise_s: 0.999
|
| 46 |
+
num_timestep_buckets: 1000
|
| 47 |
+
num_inference_timesteps: 4
|
| 48 |
+
num_target_vision_tokens: 32
|
| 49 |
+
diffusion_model_cfg: # DiT Transformers parameters
|
| 50 |
+
attention_head_dim: 48
|
| 51 |
+
cross_attention_dim: 2048 # VLM dim
|
| 52 |
+
dropout: 0.2
|
| 53 |
+
final_dropout: true
|
| 54 |
+
interleave_self_attention: true
|
| 55 |
+
norm_type: "ada_norm"
|
| 56 |
+
num_attention_heads: 32
|
| 57 |
+
num_layers: 16
|
| 58 |
+
output_dim: 1024
|
| 59 |
+
positional_embeddings: null
|
| 60 |
+
|
| 61 |
+
datasets:
|
| 62 |
+
vlm_data:
|
| 63 |
+
dataset_py: vlm_datasets
|
| 64 |
+
dataformat: llava_json
|
| 65 |
+
dataset_use: asv2_conversation_en,asv2_detailed_description_en,asv2_region_captioning_en,coco_internvl_longcap_en,coco_karpathy_train_567_en,coco_negative_gpt4o_en,coco_poetry_zh,coco_rem_en_zh,cocorem_exist_yorn_en,cocotextv2_en,cocotextv2_gpt4o_en,okvqa_en,refcoco_grounding_aug_en,refcoco_grounding_en,tallyqa_coco_en,toloka_grounding_aug_en,vqav2_en,vsr_en
|
| 66 |
+
eval_dataset: aokvqa_cauldron_llava_format
|
| 67 |
+
data_flatten: false
|
| 68 |
+
base_interval: 2
|
| 69 |
+
max_pixels: 50176
|
| 70 |
+
min_pixels: 784
|
| 71 |
+
model_max_length: 2048 # seems to be 4096 here, some related content is not processed
|
| 72 |
+
model_type: qwen2.5vl
|
| 73 |
+
per_device_batch_size: 4
|
| 74 |
+
|
| 75 |
+
vla_data:
|
| 76 |
+
dataset_py: lerobot_datasets
|
| 77 |
+
data_root_dir: playground/Datasets/YOUR_DATASET_PATH
|
| 78 |
+
data_mix: YOUR_DATASET_NAME
|
| 79 |
+
action_type: delta_ee
|
| 80 |
+
CoT_prompt: "Your task is {instruction}. To identify the key objects for your task. Locate their bounding boxes in [x1,y1,x2,y2] format."
|
| 81 |
+
CoT_answer: bbox
|
| 82 |
+
default_image_resolution: [3, 224, 224] # this is not effective
|
| 83 |
+
per_device_batch_size: 16
|
| 84 |
+
load_all_data_for_training: true # consider how to enable eval set
|
| 85 |
+
obs: ["image_0"] # this is not effective
|
| 86 |
+
image_size: [224,224] #todo is null keep raw size # this is not effective
|
| 87 |
+
|
| 88 |
+
trainer:
|
| 89 |
+
epochs: 100 # decide whether to use, he and max_train_steps conflict
|
| 90 |
+
max_train_steps: 100000
|
| 91 |
+
num_warmup_steps: 5000
|
| 92 |
+
save_interval: 5000
|
| 93 |
+
eval_interval: 100
|
| 94 |
+
learning_rate: # you can set different lr for different modules
|
| 95 |
+
base: 5e-05
|
| 96 |
+
qwen_vl_interface: 1.0e-05
|
| 97 |
+
action_model: 1.0e-04
|
| 98 |
+
lr_scheduler_type: cosine_with_min_lr
|
| 99 |
+
scheduler_specific_kwargs:
|
| 100 |
+
min_lr: 5.0e-07
|
| 101 |
+
freeze_modules: ''
|
| 102 |
+
loss_scale:
|
| 103 |
+
vla: 1.0
|
| 104 |
+
vlm: 0.1
|
| 105 |
+
# pretrained_checkpoint: ./results/Checkpoints/1_need/0601_qwenact_fixqwen_32gpus_lr_1e-3_qformer_36_37/steps_100000/pytorch_model.pt
|
| 106 |
+
# reload_modules: 'action_model,layer_qformer' # warning: this may cause model to not converge
|
| 107 |
+
repeated_diffusion_steps: 4
|
| 108 |
+
max_grad_norm: 1.0
|
| 109 |
+
warmup_ratio: 0.1
|
| 110 |
+
weight_decay: 0.0
|
| 111 |
+
logging_frequency: 10
|
| 112 |
+
gradient_clipping: 1.0
|
| 113 |
+
gradient_accumulation_steps: 1
|
| 114 |
+
|
| 115 |
+
optimizer:
|
| 116 |
+
name: AdamW
|
| 117 |
+
betas: [0.9, 0.95]
|
| 118 |
+
eps: 1.0e-08
|
| 119 |
+
weight_decay: 1.0e-08
|
| 120 |
+
|
| 121 |
+
# parameters to be determined
|
| 122 |
+
is_resume: false
|
| 123 |
+
resume_epoch: null
|
| 124 |
+
resume_step: null
|
| 125 |
+
enable_gradient_checkpointing: true
|
| 126 |
+
enable_mixed_precision_training: true
|
src/InternVLA-M1/InternVLA/config/training/internvla_cotrain_humanoid.yaml
ADDED
|
@@ -0,0 +1,128 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
run_id: humanoid_g1
|
| 2 |
+
run_root_dir: runs/InternVLA/Checkpoints
|
| 3 |
+
seed: 42
|
| 4 |
+
trackers: [jsonl, wandb]
|
| 5 |
+
wandb_entity: None
|
| 6 |
+
wandb_project: None
|
| 7 |
+
is_debug: false
|
| 8 |
+
|
| 9 |
+
framework:
|
| 10 |
+
framework_py: InternVLA-M1
|
| 11 |
+
qwenvl: # TODO later should not use qwenvl should use vlm directly
|
| 12 |
+
base_vlm: Qwen/Qwen2.5-VL-3B-Instruct
|
| 13 |
+
attn_implementation: flash_attention_2
|
| 14 |
+
vl_hidden_dim: 2048 # TODO should read from internal config
|
| 15 |
+
dino:
|
| 16 |
+
dino_backbone: dinov2_vits14
|
| 17 |
+
layer_qformer:
|
| 18 |
+
qformer_end_layer: 37
|
| 19 |
+
qformer_start_layer: 36
|
| 20 |
+
num_query_tokens: 64
|
| 21 |
+
input_dim: 2048 # here is VLM output dimension
|
| 22 |
+
ouptput_dim: 768 # here is action input dimension # here has a rule that ouptput_dim = action_hidden_dim; --> top2down logic should not default match, should be modified manually
|
| 23 |
+
grad_scale: 0.5 # let gradient pass through this module to decay, avoid destroying VLM, # seems to affect learning
|
| 24 |
+
|
| 25 |
+
action_model:
|
| 26 |
+
action_model_type: DiT-B
|
| 27 |
+
action_hidden_dim: 768
|
| 28 |
+
action_dim: 36 # 32
|
| 29 |
+
use_ema: false
|
| 30 |
+
future_action_window_size: 15 # 29
|
| 31 |
+
past_action_window_size: 0
|
| 32 |
+
repeated_diffusion_steps: 8
|
| 33 |
+
|
| 34 |
+
fm_head_config:
|
| 35 |
+
input_embedding_dim: 1536 # Action embedding after dimension 48 * 32,and DiT's attention_head_dim * attention_heads corresponds to ActionEncoder
|
| 36 |
+
hidden_size: 1024 # and DiT's last projection corresponds to ActionDecoder
|
| 37 |
+
add_pos_embed: True
|
| 38 |
+
max_seq_len: 1024
|
| 39 |
+
action_dim: 36 # 32
|
| 40 |
+
future_action_window_size: 15 # 29
|
| 41 |
+
action_horizon: 16 # 30 # = future_action_window_size + 1 + past_action_window_size
|
| 42 |
+
past_action_window_size: 0
|
| 43 |
+
noise_beta_alpha: 1.5
|
| 44 |
+
noise_beta_beta: 1.0
|
| 45 |
+
noise_s: 0.999
|
| 46 |
+
num_timestep_buckets: 1000
|
| 47 |
+
num_inference_timesteps: 4
|
| 48 |
+
num_target_vision_tokens: 32
|
| 49 |
+
diffusion_model_cfg: # DiT Transformers parameters
|
| 50 |
+
attention_head_dim: 48
|
| 51 |
+
cross_attention_dim: 2048 # VLM dim
|
| 52 |
+
dropout: 0.2
|
| 53 |
+
final_dropout: true
|
| 54 |
+
interleave_self_attention: true
|
| 55 |
+
norm_type: "ada_norm"
|
| 56 |
+
num_attention_heads: 32
|
| 57 |
+
num_layers: 16
|
| 58 |
+
output_dim: 1024
|
| 59 |
+
positional_embeddings: null
|
| 60 |
+
|
| 61 |
+
datasets:
|
| 62 |
+
vlm_data:
|
| 63 |
+
dataset_py: vlm_datasets
|
| 64 |
+
dataformat: llava_json
|
| 65 |
+
dataset_use: asv2_conversation_en,asv2_detailed_description_en,asv2_region_captioning_en,coco_internvl_longcap_en,coco_karpathy_train_567_en,coco_negative_gpt4o_en,coco_poetry_zh,coco_rem_en_zh,cocorem_exist_yorn_en,cocotextv2_en,cocotextv2_gpt4o_en,okvqa_en,refcoco_grounding_aug_en,refcoco_grounding_en,tallyqa_coco_en,toloka_grounding_aug_en,vqav2_en,vsr_en
|
| 66 |
+
eval_dataset: aokvqa_cauldron_llava_format
|
| 67 |
+
data_flatten: false
|
| 68 |
+
base_interval: 2
|
| 69 |
+
max_pixels: 50176
|
| 70 |
+
min_pixels: 784
|
| 71 |
+
model_max_length: 2048 # seems to be 4096 here, some related content is not processed
|
| 72 |
+
model_type: qwen2.5vl
|
| 73 |
+
per_device_batch_size: 4
|
| 74 |
+
|
| 75 |
+
vla_data:
|
| 76 |
+
dataset_py: lerobot_datasets
|
| 77 |
+
data_root_dir: None
|
| 78 |
+
data_mix: humanoid_
|
| 79 |
+
action_type: abs_joints
|
| 80 |
+
CoT_prompt: "Your task is {instruction}. To identify the key objects for your task. Locate their bounding boxes in [x1,y1,x2,y2] format."
|
| 81 |
+
CoT_answer: bbox
|
| 82 |
+
default_image_resolution: [3, 224, 224] # this is not effective
|
| 83 |
+
per_device_batch_size: 64 # FIXME
|
| 84 |
+
preload_all: true #FIXME
|
| 85 |
+
load_all_data_for_training: true # consider how to enable eval set
|
| 86 |
+
obs: ["image_0"] # this is not effective
|
| 87 |
+
image_size: [224,224] #todo is null keep raw size # this is not effective
|
| 88 |
+
|
| 89 |
+
trainer:
|
| 90 |
+
epochs: 100 # not effective; use max_train_steps instead
|
| 91 |
+
max_train_steps: 30000 #100000 #FIXME
|
| 92 |
+
num_warmup_steps: 0
|
| 93 |
+
save_interval: 5000 # 500
|
| 94 |
+
eval_interval: 100
|
| 95 |
+
learning_rate: # you can set different lr for different modules #FIXME
|
| 96 |
+
base: 5e-05
|
| 97 |
+
qwen_vl_interface: 1.0e-05
|
| 98 |
+
action_model: 1.0e-04
|
| 99 |
+
lr_scheduler_type: cosine_with_min_lr
|
| 100 |
+
scheduler_specific_kwargs:
|
| 101 |
+
min_lr: 5.0e-07
|
| 102 |
+
freeze_modules: 'qwen_vl_interface,layer_qformer,dino_encoder,dino_pro' # FIXME
|
| 103 |
+
loss_scale:
|
| 104 |
+
vla: 1.0
|
| 105 |
+
vlm: 0.1
|
| 106 |
+
pretrained_checkpoint: /hfm/cache/checkpoints/InternVLA-M1-Pretrain-RT-1-Bridge/checkpoints/steps_50000_pytorch_model.pt
|
| 107 |
+
# reload_modules: 'action_model,layer_qformer' # warning: this may cause model to not converge
|
| 108 |
+
skip_reload_modules: 'action_model' # skip action-related layers # FIXME
|
| 109 |
+
repeated_diffusion_steps: 4
|
| 110 |
+
max_grad_norm: 1.0
|
| 111 |
+
warmup_ratio: 0.1
|
| 112 |
+
weight_decay: 0.0
|
| 113 |
+
logging_frequency: 10
|
| 114 |
+
gradient_clipping: 1.0
|
| 115 |
+
gradient_accumulation_steps: 1
|
| 116 |
+
|
| 117 |
+
optimizer:
|
| 118 |
+
name: AdamW
|
| 119 |
+
betas: [0.9, 0.95]
|
| 120 |
+
eps: 1.0e-08
|
| 121 |
+
weight_decay: 1.0e-08
|
| 122 |
+
|
| 123 |
+
# parameters to be determined
|
| 124 |
+
is_resume: false
|
| 125 |
+
resume_epoch: null
|
| 126 |
+
resume_step: null
|
| 127 |
+
enable_gradient_checkpointing: true
|
| 128 |
+
enable_mixed_precision_training: true
|
src/InternVLA-M1/InternVLA/config/training/internvla_cotrain_libero.yaml
ADDED
|
@@ -0,0 +1,126 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
run_id: internVLA
|
| 2 |
+
run_root_dir: results/Checkpoints
|
| 3 |
+
seed: 42
|
| 4 |
+
trackers: [jsonl, wandb]
|
| 5 |
+
wandb_entity: your_wandb_entity
|
| 6 |
+
wandb_project: internVLA
|
| 7 |
+
is_debug: false
|
| 8 |
+
|
| 9 |
+
|
| 10 |
+
framework:
|
| 11 |
+
framework_py: InternVLA-M1
|
| 12 |
+
qwenvl:
|
| 13 |
+
base_vlm: ./playground/Pretrained_models/Qwen2.5-VL-3B-Instruct
|
| 14 |
+
attn_implementation: flash_attention_2
|
| 15 |
+
vl_hidden_dim: 2048
|
| 16 |
+
dino:
|
| 17 |
+
dino_backbone: dinov2_vits14
|
| 18 |
+
layer_qformer:
|
| 19 |
+
qformer_end_layer: 37
|
| 20 |
+
qformer_start_layer: 36
|
| 21 |
+
num_query_tokens: 64
|
| 22 |
+
input_dim: 2048
|
| 23 |
+
ouptput_dim: 768
|
| 24 |
+
grad_scale: 0.5
|
| 25 |
+
action_model:
|
| 26 |
+
action_model_type: DiT-B
|
| 27 |
+
action_hidden_dim: 768
|
| 28 |
+
action_dim: 7
|
| 29 |
+
use_ema: false
|
| 30 |
+
future_action_window_size: 15
|
| 31 |
+
past_action_window_size: 0
|
| 32 |
+
repeated_diffusion_steps: 8
|
| 33 |
+
|
| 34 |
+
fm_head_config:
|
| 35 |
+
input_embedding_dim: 1536 # Action embedding after dimension 48 * 32,and DiT's attention_head_dim * attention_heads corresponds to ActionEncoder
|
| 36 |
+
hidden_size: 1024 # and DiT's last projection corresponds to ActionDecoder
|
| 37 |
+
add_pos_embed: True
|
| 38 |
+
max_seq_len: 1024
|
| 39 |
+
action_dim: 7
|
| 40 |
+
future_action_window_size: 15
|
| 41 |
+
action_horizon: 16
|
| 42 |
+
past_action_window_size: 0
|
| 43 |
+
noise_beta_alpha: 1.5
|
| 44 |
+
noise_beta_beta: 1.0
|
| 45 |
+
noise_s: 0.999
|
| 46 |
+
num_timestep_buckets: 1000
|
| 47 |
+
num_inference_timesteps: 4
|
| 48 |
+
num_target_vision_tokens: 32
|
| 49 |
+
diffusion_model_cfg: # DiT Transformers parameters
|
| 50 |
+
attention_head_dim: 48
|
| 51 |
+
cross_attention_dim: 2048 # VLM dim
|
| 52 |
+
dropout: 0.2
|
| 53 |
+
final_dropout: true
|
| 54 |
+
interleave_self_attention: true
|
| 55 |
+
norm_type: "ada_norm"
|
| 56 |
+
num_attention_heads: 32
|
| 57 |
+
num_layers: 16
|
| 58 |
+
output_dim: 1024
|
| 59 |
+
positional_embeddings: null
|
| 60 |
+
|
| 61 |
+
reduce_in_full_precision: true
|
| 62 |
+
|
| 63 |
+
|
| 64 |
+
datasets:
|
| 65 |
+
vlm_data:
|
| 66 |
+
dataset_py: vlm_datasets
|
| 67 |
+
dataformat: llava_json
|
| 68 |
+
dataset_use: asv2_conversation_en,asv2_detailed_description_en,asv2_region_captioning_en,coco_internvl_longcap_en,coco_karpathy_train_567_en,coco_negative_gpt4o_en,coco_poetry_zh,coco_rem_en_zh,cocorem_exist_yorn_en,cocotextv2_en,cocotextv2_gpt4o_en,okvqa_en,refcoco_grounding_aug_en,refcoco_grounding_en,tallyqa_coco_en,toloka_grounding_aug_en,vqav2_en,vsr_en
|
| 69 |
+
eval_dataset: aokvqa_cauldron_llava_format
|
| 70 |
+
data_flatten: false
|
| 71 |
+
base_interval: 2
|
| 72 |
+
max_pixels: 12845056
|
| 73 |
+
min_pixels: 3136
|
| 74 |
+
model_max_length: 2048
|
| 75 |
+
model_type: qwen2.5vl
|
| 76 |
+
per_device_batch_size: 4
|
| 77 |
+
|
| 78 |
+
vla_data:
|
| 79 |
+
dataset_py: lerobot_datasets
|
| 80 |
+
data_root_dir: playground/Datasets/LEROBOT_LIBERO_DATA
|
| 81 |
+
data_mix: libero_goal # libero_goal
|
| 82 |
+
action_type: delta_qpos
|
| 83 |
+
CoT_prompt: "Your task is {instruction}. To identify the key objects for your task. Locate their bounding boxes in [x1,y1,x2,y2] format."
|
| 84 |
+
CoT_answer: bbox
|
| 85 |
+
default_image_resolution: [3, 224, 224]
|
| 86 |
+
per_device_batch_size: 16
|
| 87 |
+
load_all_data_for_training: true
|
| 88 |
+
obs: ["image_0"]
|
| 89 |
+
|
| 90 |
+
|
| 91 |
+
trainer:
|
| 92 |
+
epochs: 100
|
| 93 |
+
max_train_steps: 100000
|
| 94 |
+
num_warmup_steps: 5000
|
| 95 |
+
save_interval: 5000
|
| 96 |
+
eval_interval: 100
|
| 97 |
+
learning_rate:
|
| 98 |
+
base: 2.5e-05
|
| 99 |
+
qwen_vl_interface: 1.0e-05
|
| 100 |
+
action_model: 1.0e-04
|
| 101 |
+
lr_scheduler_type: cosine_with_min_lr
|
| 102 |
+
scheduler_specific_kwargs:
|
| 103 |
+
min_lr: 1.0e-06
|
| 104 |
+
freeze_modules: ''
|
| 105 |
+
loss_scale:
|
| 106 |
+
vla: 1.0
|
| 107 |
+
vlm: 0.1
|
| 108 |
+
max_grad_norm: 1.0
|
| 109 |
+
warmup_ratio: 0.1
|
| 110 |
+
weight_decay: 0.0
|
| 111 |
+
logging_frequency: 10
|
| 112 |
+
gradient_clipping: 1.0
|
| 113 |
+
gradient_accumulation_steps: 1
|
| 114 |
+
|
| 115 |
+
optimizer:
|
| 116 |
+
name: AdamW
|
| 117 |
+
betas: [0.9, 0.95]
|
| 118 |
+
eps: 1.0e-08
|
| 119 |
+
weight_decay: 1.0e-08
|
| 120 |
+
|
| 121 |
+
# parameters to be determined
|
| 122 |
+
is_resume: false
|
| 123 |
+
resume_epoch: null
|
| 124 |
+
resume_step: null
|
| 125 |
+
enable_gradient_checkpointing: true
|
| 126 |
+
enable_mixed_precision_training: true
|
src/InternVLA-M1/InternVLA/config/training/internvla_cotrain_oxe.yaml
ADDED
|
@@ -0,0 +1,100 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
run_id: qwenact
|
| 2 |
+
run_root_dir: results/Checkpoints
|
| 3 |
+
seed: 42
|
| 4 |
+
trackers: [jsonl, wandb]
|
| 5 |
+
wandb_entity: your_wandb_entity
|
| 6 |
+
wandb_project: internVLAAA
|
| 7 |
+
is_debug: false
|
| 8 |
+
|
| 9 |
+
framework:
|
| 10 |
+
framework_py: InternVLA-M1
|
| 11 |
+
qwenvl:
|
| 12 |
+
base_vlm: ./playground/Pretrained_models/Qwen2.5-VL-3B-Instruct
|
| 13 |
+
attn_implementation: flash_attention_2
|
| 14 |
+
vl_hidden_dim: 2048
|
| 15 |
+
dino:
|
| 16 |
+
dino_backbone: dinov2_vits14
|
| 17 |
+
layer_qformer:
|
| 18 |
+
qformer_end_layer: 37
|
| 19 |
+
qformer_start_layer: 36
|
| 20 |
+
num_query_tokens: 64
|
| 21 |
+
input_dim: 2048
|
| 22 |
+
ouptput_dim: 768
|
| 23 |
+
grad_scale: 0.5
|
| 24 |
+
|
| 25 |
+
action_model:
|
| 26 |
+
action_model_type: DiT-B
|
| 27 |
+
action_hidden_dim: 768
|
| 28 |
+
action_dim: 7
|
| 29 |
+
use_ema: false
|
| 30 |
+
future_action_window_size: 15
|
| 31 |
+
past_action_window_size: 0
|
| 32 |
+
repeated_diffusion_steps: 8
|
| 33 |
+
|
| 34 |
+
|
| 35 |
+
datasets:
|
| 36 |
+
vlm_data:
|
| 37 |
+
dataset_py: vlm_datasets
|
| 38 |
+
dataformat: llava_json
|
| 39 |
+
dataset_use: asv2_conversation_en,asv2_detailed_description_en,asv2_region_captioning_en,coco_internvl_longcap_en,coco_karpathy_train_567_en,coco_negative_gpt4o_en,coco_poetry_zh,coco_rem_en_zh,cocorem_exist_yorn_en,cocotextv2_en,cocotextv2_gpt4o_en,okvqa_en,refcoco_grounding_aug_en,refcoco_grounding_en,tallyqa_coco_en,toloka_grounding_aug_en,vqav2_en,vsr_en
|
| 40 |
+
eval_dataset: aokvqa_cauldron_llava_format
|
| 41 |
+
data_flatten: false
|
| 42 |
+
base_interval: 2
|
| 43 |
+
max_pixels: 50176
|
| 44 |
+
min_pixels: 784
|
| 45 |
+
model_max_length: 2048
|
| 46 |
+
model_type: qwen2.5vl
|
| 47 |
+
per_device_batch_size: 4
|
| 48 |
+
|
| 49 |
+
vla_data:
|
| 50 |
+
dataset_py: lerobot_datasets
|
| 51 |
+
data_root_dir: playground/Datasets/OXE_LEROBOT_DATASET
|
| 52 |
+
data_mix: bridge_rt_1 # bridge_rt_1
|
| 53 |
+
action_type: delta_ee
|
| 54 |
+
CoT_prompt: "Your task is {instruction}. To identify the key objects for your task. Locate their bounding boxes in [x1,y1,x2,y2] format."
|
| 55 |
+
CoT_answer: bbox
|
| 56 |
+
default_image_resolution: [3, 224, 224]
|
| 57 |
+
per_device_batch_size: 16
|
| 58 |
+
load_all_data_for_training: true
|
| 59 |
+
obs: ["image_0"]
|
| 60 |
+
image_size: [224,224]
|
| 61 |
+
|
| 62 |
+
trainer:
|
| 63 |
+
epochs: 100
|
| 64 |
+
max_train_steps: 100000
|
| 65 |
+
num_warmup_steps: 5000
|
| 66 |
+
save_interval: 5000
|
| 67 |
+
eval_interval: 100
|
| 68 |
+
learning_rate:
|
| 69 |
+
base: 1e-05
|
| 70 |
+
qwen_vl_interface: 1.0e-05
|
| 71 |
+
action_model: 1.0e-04
|
| 72 |
+
lr_scheduler_type: cosine_with_min_lr
|
| 73 |
+
scheduler_specific_kwargs:
|
| 74 |
+
min_lr: 5.0e-07
|
| 75 |
+
freeze_modules: ''
|
| 76 |
+
loss_scale:
|
| 77 |
+
vla: 1.0
|
| 78 |
+
vlm: 0.1
|
| 79 |
+
# pretrained_checkpoint: ./results/Checkpoints/1_need/0601_qwenact_fixqwen_32gpus_lr_1e-3_qformer_36_37/steps_100000/pytorch_model.pt
|
| 80 |
+
# reload_modules: 'action_model,layer_qformer' # warning: this may cause model to not converge
|
| 81 |
+
repeated_diffusion_steps: 4
|
| 82 |
+
max_grad_norm: 1.0
|
| 83 |
+
warmup_ratio: 0.1
|
| 84 |
+
weight_decay: 0.0
|
| 85 |
+
logging_frequency: 10
|
| 86 |
+
gradient_clipping: 1.0
|
| 87 |
+
gradient_accumulation_steps: 1
|
| 88 |
+
|
| 89 |
+
optimizer:
|
| 90 |
+
name: AdamW
|
| 91 |
+
betas: [0.9, 0.95]
|
| 92 |
+
eps: 1.0e-08
|
| 93 |
+
weight_decay: 1.0e-08
|
| 94 |
+
|
| 95 |
+
# parameters to be determined
|
| 96 |
+
is_resume: false
|
| 97 |
+
resume_epoch: null
|
| 98 |
+
resume_step: null
|
| 99 |
+
enable_gradient_checkpointing: true
|
| 100 |
+
enable_mixed_precision_training: true
|
src/InternVLA-M1/InternVLA/config/training/internvla_cotrain_sim_demo.yaml
ADDED
|
@@ -0,0 +1,124 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
run_id: qwenact
|
| 2 |
+
run_root_dir: results/Checkpoints
|
| 3 |
+
seed: 42
|
| 4 |
+
trackers: [jsonl, wandb]
|
| 5 |
+
wandb_entity: ailab-manipulation
|
| 6 |
+
wandb_project: internVLA
|
| 7 |
+
is_debug: false
|
| 8 |
+
|
| 9 |
+
framework:
|
| 10 |
+
framework_py: InternVLA-M1
|
| 11 |
+
qwenvl: # TODO later should not use qwenvl should use vlm directly
|
| 12 |
+
base_vlm: ./playground/Pretrained_models/Qwen2.5-VL-3B-Instruct
|
| 13 |
+
attn_implementation: flash_attention_2
|
| 14 |
+
vl_hidden_dim: 2048 # TODO should read from internal config
|
| 15 |
+
dino:
|
| 16 |
+
dino_backbone: dinov2_vits14
|
| 17 |
+
layer_qformer:
|
| 18 |
+
qformer_end_layer: 37
|
| 19 |
+
qformer_start_layer: 36
|
| 20 |
+
num_query_tokens: 64
|
| 21 |
+
input_dim: 2048 # here is VLM output dimension
|
| 22 |
+
ouptput_dim: 768 # here is action input dimension # here has a rule that ouptput_dim = action_hidden_dim; --> top2down logic should not default match, should be modified manually
|
| 23 |
+
grad_scale: 0.5 # let gradient pass through this module to decay, avoid destroying VLM, # seems to affect learning
|
| 24 |
+
|
| 25 |
+
action_model:
|
| 26 |
+
action_model_type: DiT-B
|
| 27 |
+
action_hidden_dim: 768
|
| 28 |
+
action_dim: 8
|
| 29 |
+
use_ema: false
|
| 30 |
+
future_action_window_size: 15
|
| 31 |
+
past_action_window_size: 0
|
| 32 |
+
repeated_diffusion_steps: 8
|
| 33 |
+
|
| 34 |
+
fm_head_config:
|
| 35 |
+
input_embedding_dim: 1536 # Action embedding after dimension 48 * 32,and DiT's attention_head_dim * attention_heads corresponds to ActionEncoder
|
| 36 |
+
hidden_size: 1024 # and DiT's last projection corresponds to ActionDecoder
|
| 37 |
+
add_pos_embed: True
|
| 38 |
+
max_seq_len: 1024
|
| 39 |
+
action_dim: 8
|
| 40 |
+
future_action_window_size: 15
|
| 41 |
+
action_horizon: 16 # = future_action_window_size + 1 + past_action_window_size
|
| 42 |
+
past_action_window_size: 0
|
| 43 |
+
noise_beta_alpha: 1.5
|
| 44 |
+
noise_beta_beta: 1.0
|
| 45 |
+
noise_s: 0.999
|
| 46 |
+
num_timestep_buckets: 1000
|
| 47 |
+
num_inference_timesteps: 4
|
| 48 |
+
num_target_vision_tokens: 32
|
| 49 |
+
diffusion_model_cfg: # DiT Transformers parameters
|
| 50 |
+
attention_head_dim: 48
|
| 51 |
+
cross_attention_dim: 2048 # VLM dim
|
| 52 |
+
dropout: 0.2
|
| 53 |
+
final_dropout: true
|
| 54 |
+
interleave_self_attention: true
|
| 55 |
+
norm_type: "ada_norm"
|
| 56 |
+
num_attention_heads: 32
|
| 57 |
+
num_layers: 16
|
| 58 |
+
output_dim: 1024
|
| 59 |
+
positional_embeddings: null
|
| 60 |
+
|
| 61 |
+
datasets:
|
| 62 |
+
vlm_data:
|
| 63 |
+
dataset_py: vlm_datasets
|
| 64 |
+
dataformat: llava_json
|
| 65 |
+
dataset_use: asv2_conversation_en,asv2_detailed_description_en,asv2_region_captioning_en,coco_internvl_longcap_en,coco_karpathy_train_567_en,coco_negative_gpt4o_en,coco_poetry_zh,coco_rem_en_zh,cocorem_exist_yorn_en,cocotextv2_en,cocotextv2_gpt4o_en,okvqa_en,refcoco_grounding_aug_en,refcoco_grounding_en,tallyqa_coco_en,toloka_grounding_aug_en,vqav2_en,vsr_en
|
| 66 |
+
eval_dataset: aokvqa_cauldron_llava_format
|
| 67 |
+
data_flatten: false
|
| 68 |
+
base_interval: 2
|
| 69 |
+
max_pixels: 50176
|
| 70 |
+
min_pixels: 784
|
| 71 |
+
model_max_length: 2048 # seems to be 4096 here, some related content is not processed
|
| 72 |
+
model_type: qwen2.5vl
|
| 73 |
+
per_device_batch_size: 4
|
| 74 |
+
|
| 75 |
+
vla_data:
|
| 76 |
+
dataset_py: lerobot_datasets
|
| 77 |
+
data_root_dir: playground/demo_data
|
| 78 |
+
data_mix: demo_sim_pick_place
|
| 79 |
+
action_type: delta_joints
|
| 80 |
+
CoT_prompt: "Your task is {instruction}. To identify the key objects for your task. Locate their bounding boxes in [x1,y1,x2,y2] format."
|
| 81 |
+
CoT_answer: bbox
|
| 82 |
+
default_image_resolution: [3, 224, 224] # this is not effective
|
| 83 |
+
per_device_batch_size: 16
|
| 84 |
+
load_all_data_for_training: true # consider how to enable eval set
|
| 85 |
+
obs: ["image_0"] # this is not effective
|
| 86 |
+
image_size: [224,224] #todo is null keep raw size # this is not effective
|
| 87 |
+
|
| 88 |
+
trainer:
|
| 89 |
+
epochs: 100 # decide whether to use, he and max_train_steps conflict
|
| 90 |
+
max_train_steps: 100000
|
| 91 |
+
num_warmup_steps: 5000
|
| 92 |
+
save_interval: 5000
|
| 93 |
+
eval_interval: 100
|
| 94 |
+
learning_rate: # you can set different lr for different modules
|
| 95 |
+
base: 5e-05
|
| 96 |
+
qwen_vl_interface: 1.0e-05
|
| 97 |
+
action_model: 1.0e-04
|
| 98 |
+
lr_scheduler_type: cosine_with_min_lr
|
| 99 |
+
scheduler_specific_kwargs:
|
| 100 |
+
min_lr: 5.0e-07
|
| 101 |
+
freeze_modules: ''
|
| 102 |
+
loss_scale:
|
| 103 |
+
vla: 1.0
|
| 104 |
+
vlm: 0.1
|
| 105 |
+
repeated_diffusion_steps: 4
|
| 106 |
+
max_grad_norm: 1.0
|
| 107 |
+
warmup_ratio: 0.1
|
| 108 |
+
weight_decay: 0.0
|
| 109 |
+
logging_frequency: 10
|
| 110 |
+
gradient_clipping: 1.0
|
| 111 |
+
gradient_accumulation_steps: 1
|
| 112 |
+
|
| 113 |
+
optimizer:
|
| 114 |
+
name: AdamW
|
| 115 |
+
betas: [0.9, 0.95]
|
| 116 |
+
eps: 1.0e-08
|
| 117 |
+
weight_decay: 1.0e-08
|
| 118 |
+
|
| 119 |
+
# parameters to be determined
|
| 120 |
+
is_resume: false
|
| 121 |
+
resume_epoch: null
|
| 122 |
+
resume_step: null
|
| 123 |
+
enable_gradient_checkpointing: true
|
| 124 |
+
enable_mixed_precision_training: true
|
src/InternVLA-M1/InternVLA/dataloader/__init__.py
ADDED
|
@@ -0,0 +1,63 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import json
|
| 2 |
+
import os
|
| 3 |
+
from accelerate.logging import get_logger
|
| 4 |
+
import numpy as np
|
| 5 |
+
from torch.utils.data import DataLoader
|
| 6 |
+
import numpy as np
|
| 7 |
+
import torch.distributed as dist
|
| 8 |
+
from pathlib import Path
|
| 9 |
+
from InternVLA.dataloader.vlm_datasets import make_vlm_dataloader
|
| 10 |
+
|
| 11 |
+
logger = get_logger(__name__)
|
| 12 |
+
|
| 13 |
+
def save_dataset_statistics(dataset_statistics, run_dir):
|
| 14 |
+
"""Saves a `dataset_statistics.json` file."""
|
| 15 |
+
out_path = run_dir / "dataset_statistics.json"
|
| 16 |
+
with open(out_path, "w") as f_json:
|
| 17 |
+
for _, stats in dataset_statistics.items():
|
| 18 |
+
for k in stats["action"].keys():
|
| 19 |
+
if isinstance(stats["action"][k], np.ndarray):
|
| 20 |
+
stats["action"][k] = stats["action"][k].tolist()
|
| 21 |
+
if "proprio" in stats:
|
| 22 |
+
for k in stats["proprio"].keys():
|
| 23 |
+
if isinstance(stats["proprio"][k], np.ndarray):
|
| 24 |
+
stats["proprio"][k] = stats["proprio"][k].tolist()
|
| 25 |
+
if "num_trajectories" in stats:
|
| 26 |
+
if isinstance(stats["num_trajectories"], np.ndarray):
|
| 27 |
+
stats["num_trajectories"] = stats["num_trajectories"].item()
|
| 28 |
+
if "num_transitions" in stats:
|
| 29 |
+
if isinstance(stats["num_transitions"], np.ndarray):
|
| 30 |
+
stats["num_transitions"] = stats["num_transitions"].item()
|
| 31 |
+
json.dump(dataset_statistics, f_json, indent=2)
|
| 32 |
+
logger.info(f"Saved dataset statistics file at path {out_path}")
|
| 33 |
+
|
| 34 |
+
|
| 35 |
+
|
| 36 |
+
def build_dataloader(cfg, dataset_py="lerobot_datasets_oxe"): # TODO now here only is get dataset, we need mv dataloader to here
|
| 37 |
+
|
| 38 |
+
if dataset_py == "lerobot_datasets":
|
| 39 |
+
from InternVLA.dataloader.lerobot_datasets import get_vla_dataset, collate_fn
|
| 40 |
+
vla_dataset_cfg = cfg.datasets.vla_data
|
| 41 |
+
|
| 42 |
+
data_root_dir = vla_dataset_cfg.data_root_dir
|
| 43 |
+
data_mix = vla_dataset_cfg.data_mix
|
| 44 |
+
|
| 45 |
+
vla_dataset = get_vla_dataset(data_cfg=vla_dataset_cfg)
|
| 46 |
+
|
| 47 |
+
vla_train_dataloader = DataLoader(
|
| 48 |
+
vla_dataset,
|
| 49 |
+
batch_size=cfg.datasets.vla_data.per_device_batch_size,
|
| 50 |
+
collate_fn=collate_fn,
|
| 51 |
+
num_workers=8,
|
| 52 |
+
# shuffle=True
|
| 53 |
+
)
|
| 54 |
+
if dist.get_rank() == 0:
|
| 55 |
+
|
| 56 |
+
output_dir = Path(cfg.output_dir)
|
| 57 |
+
vla_dataset.save_dataset_statistics(output_dir / "dataset_statistics.json")
|
| 58 |
+
return vla_train_dataloader
|
| 59 |
+
elif dataset_py == "vlm_datasets":
|
| 60 |
+
vlm_data_module = make_vlm_dataloader(cfg)
|
| 61 |
+
vlm_train_dataloader = vlm_data_module["train_dataloader"]
|
| 62 |
+
|
| 63 |
+
return vlm_train_dataloader
|
src/InternVLA-M1/InternVLA/dataloader/gr00t_lerobot/README.md
ADDED
|
File without changes
|
src/InternVLA-M1/InternVLA/dataloader/gr00t_lerobot/__init__.py
ADDED
|
File without changes
|
src/InternVLA-M1/InternVLA/dataloader/gr00t_lerobot/data_config.py
ADDED
|
@@ -0,0 +1,770 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
| 2 |
+
# SPDX-License-Identifier: Apache-2.0
|
| 3 |
+
#
|
| 4 |
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
| 5 |
+
# you may not use this file except in compliance with the License.
|
| 6 |
+
# You may obtain a copy of the License at
|
| 7 |
+
#
|
| 8 |
+
# http://www.apache.org/licenses/LICENSE-2.0
|
| 9 |
+
#
|
| 10 |
+
# Unless required by applicable law or agreed to in writing, software
|
| 11 |
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
| 12 |
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
| 13 |
+
# See the License for the specific language governing permissions and
|
| 14 |
+
# limitations under the License.
|
| 15 |
+
|
| 16 |
+
from abc import ABC, abstractmethod
|
| 17 |
+
|
| 18 |
+
from InternVLA.dataloader.gr00t_lerobot.datasets import ModalityConfig
|
| 19 |
+
from InternVLA.dataloader.gr00t_lerobot.transform.base import ComposedModalityTransform, ModalityTransform
|
| 20 |
+
from InternVLA.dataloader.gr00t_lerobot.transform.concat import ConcatTransform
|
| 21 |
+
from InternVLA.dataloader.gr00t_lerobot.transform.state_action import (
|
| 22 |
+
StateActionSinCosTransform,
|
| 23 |
+
StateActionToTensor,
|
| 24 |
+
StateActionTransform,
|
| 25 |
+
)
|
| 26 |
+
from InternVLA.dataloader.gr00t_lerobot.transform.video import (
|
| 27 |
+
VideoColorJitter,
|
| 28 |
+
VideoCrop,
|
| 29 |
+
VideoResize,
|
| 30 |
+
VideoToNumpy,
|
| 31 |
+
VideoToTensor,
|
| 32 |
+
)
|
| 33 |
+
# from gr00t.model.transforms import GR00TTransform
|
| 34 |
+
|
| 35 |
+
|
| 36 |
+
class BaseDataConfig(ABC):
|
| 37 |
+
@abstractmethod
|
| 38 |
+
def modality_config(self) -> dict[str, ModalityConfig]:
|
| 39 |
+
pass
|
| 40 |
+
|
| 41 |
+
@abstractmethod
|
| 42 |
+
def transform(self) -> ModalityTransform:
|
| 43 |
+
pass
|
| 44 |
+
|
| 45 |
+
|
| 46 |
+
###########################################################################################
|
| 47 |
+
|
| 48 |
+
class OxeDroidDataConfig:
|
| 49 |
+
video_keys = [
|
| 50 |
+
"video.exterior_image_1",
|
| 51 |
+
"video.exterior_image_2",
|
| 52 |
+
"video.wrist_image",
|
| 53 |
+
]
|
| 54 |
+
state_keys = [
|
| 55 |
+
"state.eef_position",
|
| 56 |
+
"state.eef_rotation",
|
| 57 |
+
"state.gripper_position",
|
| 58 |
+
]
|
| 59 |
+
action_keys = [
|
| 60 |
+
"action.eef_position_delta",
|
| 61 |
+
"action.eef_rotation_delta",
|
| 62 |
+
"action.gripper_position",
|
| 63 |
+
]
|
| 64 |
+
language_keys = ["annotation.language.language_instruction"]
|
| 65 |
+
observation_indices = [0]
|
| 66 |
+
action_indices = list(range(16))
|
| 67 |
+
|
| 68 |
+
def modality_config(self):
|
| 69 |
+
video_modality = ModalityConfig(
|
| 70 |
+
delta_indices=self.observation_indices,
|
| 71 |
+
modality_keys=self.video_keys,
|
| 72 |
+
)
|
| 73 |
+
state_modality = ModalityConfig(
|
| 74 |
+
delta_indices=self.observation_indices,
|
| 75 |
+
modality_keys=self.state_keys,
|
| 76 |
+
)
|
| 77 |
+
action_modality = ModalityConfig(
|
| 78 |
+
delta_indices=self.action_indices,
|
| 79 |
+
modality_keys=self.action_keys,
|
| 80 |
+
)
|
| 81 |
+
language_modality = ModalityConfig(
|
| 82 |
+
delta_indices=self.observation_indices,
|
| 83 |
+
modality_keys=self.language_keys,
|
| 84 |
+
)
|
| 85 |
+
modality_configs = {
|
| 86 |
+
"video": video_modality,
|
| 87 |
+
"state": state_modality,
|
| 88 |
+
"action": action_modality,
|
| 89 |
+
"language": language_modality,
|
| 90 |
+
}
|
| 91 |
+
return modality_configs
|
| 92 |
+
|
| 93 |
+
def transform(self):
|
| 94 |
+
transforms = [
|
| 95 |
+
# video transforms
|
| 96 |
+
VideoToTensor(apply_to=self.video_keys),
|
| 97 |
+
VideoCrop(apply_to=self.video_keys, scale=0.95),
|
| 98 |
+
VideoResize(apply_to=self.video_keys, height=224, width=224, interpolation="linear"),
|
| 99 |
+
VideoColorJitter(
|
| 100 |
+
apply_to=self.video_keys,
|
| 101 |
+
brightness=0.3,
|
| 102 |
+
contrast=0.4,
|
| 103 |
+
saturation=0.5,
|
| 104 |
+
hue=0.08,
|
| 105 |
+
),
|
| 106 |
+
VideoToNumpy(apply_to=self.video_keys),
|
| 107 |
+
# state transforms
|
| 108 |
+
StateActionToTensor(apply_to=self.state_keys),
|
| 109 |
+
StateActionTransform(
|
| 110 |
+
apply_to=self.state_keys,
|
| 111 |
+
normalization_modes={
|
| 112 |
+
"state.eef_position": "min_max",
|
| 113 |
+
"state.gripper_position": "min_max",
|
| 114 |
+
},
|
| 115 |
+
target_rotations={
|
| 116 |
+
"state.eef_rotation": "rotation_6d",
|
| 117 |
+
},
|
| 118 |
+
),
|
| 119 |
+
# action transforms
|
| 120 |
+
StateActionToTensor(apply_to=self.action_keys),
|
| 121 |
+
StateActionTransform(
|
| 122 |
+
apply_to=self.action_keys,
|
| 123 |
+
normalization_modes={
|
| 124 |
+
"action.gripper_position": "binary",
|
| 125 |
+
},
|
| 126 |
+
target_rotations={"action.eef_rotation_delta": "axis_angle"},
|
| 127 |
+
),
|
| 128 |
+
# concat transforms
|
| 129 |
+
ConcatTransform(
|
| 130 |
+
video_concat_order=self.video_keys,
|
| 131 |
+
state_concat_order=self.state_keys,
|
| 132 |
+
action_concat_order=self.action_keys,
|
| 133 |
+
),
|
| 134 |
+
GR00TTransform(
|
| 135 |
+
state_horizon=len(self.observation_indices),
|
| 136 |
+
action_horizon=len(self.action_indices),
|
| 137 |
+
max_state_dim=64,
|
| 138 |
+
max_action_dim=32,
|
| 139 |
+
),
|
| 140 |
+
]
|
| 141 |
+
|
| 142 |
+
return ComposedModalityTransform(transforms=transforms)
|
| 143 |
+
|
| 144 |
+
|
| 145 |
+
###########################################################################################
|
| 146 |
+
|
| 147 |
+
|
| 148 |
+
class OxeBridgeDataConfig:
|
| 149 |
+
video_keys = [
|
| 150 |
+
"video.image_0",
|
| 151 |
+
]
|
| 152 |
+
state_keys = [
|
| 153 |
+
"state.x",
|
| 154 |
+
"state.y",
|
| 155 |
+
"state.z",
|
| 156 |
+
"state.roll",
|
| 157 |
+
"state.pitch",
|
| 158 |
+
"state.yaw",
|
| 159 |
+
"state.pad",
|
| 160 |
+
"state.gripper",
|
| 161 |
+
]
|
| 162 |
+
action_keys = [
|
| 163 |
+
"action.x",
|
| 164 |
+
"action.y",
|
| 165 |
+
"action.z",
|
| 166 |
+
"action.roll",
|
| 167 |
+
"action.pitch",
|
| 168 |
+
"action.yaw",
|
| 169 |
+
"action.gripper",
|
| 170 |
+
]
|
| 171 |
+
language_keys = ["annotation.human.action.task_description"]
|
| 172 |
+
observation_indices = [0]
|
| 173 |
+
action_indices = list(range(16))
|
| 174 |
+
|
| 175 |
+
def modality_config(self):
|
| 176 |
+
video_modality = ModalityConfig(
|
| 177 |
+
delta_indices=self.observation_indices,
|
| 178 |
+
modality_keys=self.video_keys,
|
| 179 |
+
)
|
| 180 |
+
state_modality = ModalityConfig(
|
| 181 |
+
delta_indices=self.observation_indices,
|
| 182 |
+
modality_keys=self.state_keys,
|
| 183 |
+
)
|
| 184 |
+
action_modality = ModalityConfig(
|
| 185 |
+
delta_indices=self.action_indices,
|
| 186 |
+
modality_keys=self.action_keys,
|
| 187 |
+
)
|
| 188 |
+
language_modality = ModalityConfig(
|
| 189 |
+
delta_indices=self.observation_indices,
|
| 190 |
+
modality_keys=self.language_keys,
|
| 191 |
+
)
|
| 192 |
+
modality_configs = {
|
| 193 |
+
"video": video_modality,
|
| 194 |
+
"state": state_modality,
|
| 195 |
+
"action": action_modality,
|
| 196 |
+
"language": language_modality,
|
| 197 |
+
}
|
| 198 |
+
return modality_configs
|
| 199 |
+
|
| 200 |
+
def transform(self):
|
| 201 |
+
transforms = [
|
| 202 |
+
# video transforms
|
| 203 |
+
# VideoToTensor(apply_to=self.video_keys),
|
| 204 |
+
# VideoCrop(apply_to=self.video_keys, scale=0.95),
|
| 205 |
+
# VideoResize(apply_to=self.video_keys, height=224, width=224, interpolation="linear"),
|
| 206 |
+
# VideoColorJitter(
|
| 207 |
+
# apply_to=self.video_keys,
|
| 208 |
+
# brightness=0.3,
|
| 209 |
+
# contrast=0.4,
|
| 210 |
+
# saturation=0.5,
|
| 211 |
+
# hue=0.08,
|
| 212 |
+
# ),
|
| 213 |
+
# VideoToNumpy(apply_to=self.video_keys),
|
| 214 |
+
# state transforms
|
| 215 |
+
StateActionToTensor(apply_to=self.state_keys),
|
| 216 |
+
StateActionTransform(
|
| 217 |
+
apply_to=self.state_keys,
|
| 218 |
+
normalization_modes={
|
| 219 |
+
"state.x": "q99",
|
| 220 |
+
"state.y": "q99",
|
| 221 |
+
"state.z": "q99",
|
| 222 |
+
"state.roll": "q99",
|
| 223 |
+
"state.pitch": "q99",
|
| 224 |
+
"state.yaw": "q99",
|
| 225 |
+
"state.pad": "q99",
|
| 226 |
+
"state.gripper": "binary",
|
| 227 |
+
},
|
| 228 |
+
),
|
| 229 |
+
# action transforms
|
| 230 |
+
StateActionToTensor(apply_to=self.action_keys),
|
| 231 |
+
StateActionTransform(
|
| 232 |
+
apply_to=self.action_keys,
|
| 233 |
+
normalization_modes={
|
| 234 |
+
"action.x": "q99",
|
| 235 |
+
"action.y": "q99",
|
| 236 |
+
"action.z": "q99",
|
| 237 |
+
"action.roll": "q99",
|
| 238 |
+
"action.pitch": "q99",
|
| 239 |
+
"action.yaw": "q99",
|
| 240 |
+
"action.gripper": "binary",
|
| 241 |
+
},
|
| 242 |
+
),
|
| 243 |
+
# concat transforms
|
| 244 |
+
# ConcatTransform(
|
| 245 |
+
# # video_concat_order=self.video_keys,
|
| 246 |
+
# state_concat_order=self.state_keys,
|
| 247 |
+
# action_concat_order=self.action_keys,
|
| 248 |
+
# ),
|
| 249 |
+
# GR00TTransform(
|
| 250 |
+
# state_horizon=len(self.observation_indices),
|
| 251 |
+
# action_horizon=len(self.action_indices),
|
| 252 |
+
# max_state_dim=64,
|
| 253 |
+
# max_action_dim=32,
|
| 254 |
+
# ),
|
| 255 |
+
]
|
| 256 |
+
|
| 257 |
+
return ComposedModalityTransform(transforms=transforms)
|
| 258 |
+
|
| 259 |
+
|
| 260 |
+
###########################################################################################
|
| 261 |
+
|
| 262 |
+
class OxeRT1DataConfig:
|
| 263 |
+
video_keys = [
|
| 264 |
+
"video.image",
|
| 265 |
+
]
|
| 266 |
+
state_keys = [
|
| 267 |
+
"state.x",
|
| 268 |
+
"state.y",
|
| 269 |
+
"state.z",
|
| 270 |
+
"state.rx",
|
| 271 |
+
"state.ry",
|
| 272 |
+
"state.rz",
|
| 273 |
+
"state.rw",
|
| 274 |
+
"state.gripper",
|
| 275 |
+
]
|
| 276 |
+
action_keys = [
|
| 277 |
+
"action.x",
|
| 278 |
+
"action.y",
|
| 279 |
+
"action.z",
|
| 280 |
+
"action.roll",
|
| 281 |
+
"action.pitch",
|
| 282 |
+
"action.yaw",
|
| 283 |
+
"action.gripper",
|
| 284 |
+
]
|
| 285 |
+
language_keys = ["annotation.human.action.task_description"]
|
| 286 |
+
observation_indices = [0]
|
| 287 |
+
action_indices = list(range(16))
|
| 288 |
+
|
| 289 |
+
def modality_config(self):
|
| 290 |
+
video_modality = ModalityConfig(
|
| 291 |
+
delta_indices=self.observation_indices,
|
| 292 |
+
modality_keys=self.video_keys,
|
| 293 |
+
)
|
| 294 |
+
state_modality = ModalityConfig(
|
| 295 |
+
delta_indices=self.observation_indices,
|
| 296 |
+
modality_keys=self.state_keys,
|
| 297 |
+
)
|
| 298 |
+
action_modality = ModalityConfig(
|
| 299 |
+
delta_indices=self.action_indices,
|
| 300 |
+
modality_keys=self.action_keys,
|
| 301 |
+
)
|
| 302 |
+
language_modality = ModalityConfig(
|
| 303 |
+
delta_indices=self.observation_indices,
|
| 304 |
+
modality_keys=self.language_keys,
|
| 305 |
+
)
|
| 306 |
+
modality_configs = {
|
| 307 |
+
"video": video_modality,
|
| 308 |
+
"state": state_modality,
|
| 309 |
+
"action": action_modality,
|
| 310 |
+
"language": language_modality,
|
| 311 |
+
}
|
| 312 |
+
return modality_configs
|
| 313 |
+
|
| 314 |
+
def transform(self):
|
| 315 |
+
transforms = [
|
| 316 |
+
# video transforms
|
| 317 |
+
# VideoToTensor(apply_to=self.video_keys),
|
| 318 |
+
# VideoCrop(apply_to=self.video_keys, scale=0.95),
|
| 319 |
+
# VideoResize(apply_to=self.video_keys, height=224, width=224, interpolation="linear"),
|
| 320 |
+
# VideoColorJitter(
|
| 321 |
+
# apply_to=self.video_keys,
|
| 322 |
+
# brightness=0.3,
|
| 323 |
+
# contrast=0.4,
|
| 324 |
+
# saturation=0.5,
|
| 325 |
+
# hue=0.08,
|
| 326 |
+
# ),
|
| 327 |
+
# VideoToNumpy(apply_to=self.video_keys),
|
| 328 |
+
# state transforms
|
| 329 |
+
StateActionToTensor(apply_to=self.state_keys),
|
| 330 |
+
StateActionTransform(
|
| 331 |
+
apply_to=self.state_keys,
|
| 332 |
+
normalization_modes={
|
| 333 |
+
"state.x": "q99",
|
| 334 |
+
"state.y": "q99",
|
| 335 |
+
"state.z": "q99",
|
| 336 |
+
"state.rx": "q99",
|
| 337 |
+
"state.ry": "q99",
|
| 338 |
+
"state.rz": "q99",
|
| 339 |
+
"state.rw": "q99",
|
| 340 |
+
"state.gripper": "binary",
|
| 341 |
+
},
|
| 342 |
+
),
|
| 343 |
+
# action transforms
|
| 344 |
+
StateActionToTensor(apply_to=self.action_keys),
|
| 345 |
+
StateActionTransform(
|
| 346 |
+
apply_to=self.action_keys,
|
| 347 |
+
normalization_modes={
|
| 348 |
+
"action.x": "q99",
|
| 349 |
+
"action.y": "q99",
|
| 350 |
+
"action.z": "q99",
|
| 351 |
+
"action.roll": "q99",
|
| 352 |
+
"action.pitch": "q99",
|
| 353 |
+
"action.yaw": "q99",
|
| 354 |
+
"action.gripper": "binary",
|
| 355 |
+
},
|
| 356 |
+
),
|
| 357 |
+
# concat transforms
|
| 358 |
+
# ConcatTransform(
|
| 359 |
+
# # video_concat_order=self.video_keys,
|
| 360 |
+
# state_concat_order=self.state_keys,
|
| 361 |
+
# action_concat_order=self.action_keys,
|
| 362 |
+
# ),
|
| 363 |
+
# GR00TTransform(
|
| 364 |
+
# state_horizon=len(self.observation_indices),
|
| 365 |
+
# action_horizon=len(self.action_indices),
|
| 366 |
+
# max_state_dim=64,
|
| 367 |
+
# max_action_dim=32,
|
| 368 |
+
# ),
|
| 369 |
+
]
|
| 370 |
+
|
| 371 |
+
return ComposedModalityTransform(transforms=transforms)
|
| 372 |
+
|
| 373 |
+
|
| 374 |
+
###########################################################################################
|
| 375 |
+
|
| 376 |
+
|
| 377 |
+
class SingleFrankaRobotiqDeltaEefDataConfig:
|
| 378 |
+
video_keys = [
|
| 379 |
+
"video.base_view",
|
| 380 |
+
"video.ego_view",
|
| 381 |
+
]
|
| 382 |
+
state_keys = [
|
| 383 |
+
"state.eef_position",
|
| 384 |
+
"state.eef_rotation",
|
| 385 |
+
]
|
| 386 |
+
action_keys = [
|
| 387 |
+
"action.delta_eef_position",
|
| 388 |
+
"action.delta_eef_rotation",
|
| 389 |
+
"action.gripper_close",
|
| 390 |
+
]
|
| 391 |
+
|
| 392 |
+
language_keys = ["annotation.human.action.task_description"]
|
| 393 |
+
observation_indices = [0]
|
| 394 |
+
action_indices = list(range(16))
|
| 395 |
+
|
| 396 |
+
def modality_config(self):
|
| 397 |
+
video_modality = ModalityConfig(
|
| 398 |
+
delta_indices=self.observation_indices,
|
| 399 |
+
modality_keys=self.video_keys,
|
| 400 |
+
)
|
| 401 |
+
state_modality = ModalityConfig(
|
| 402 |
+
delta_indices=self.observation_indices,
|
| 403 |
+
modality_keys=self.state_keys,
|
| 404 |
+
)
|
| 405 |
+
action_modality = ModalityConfig(
|
| 406 |
+
delta_indices=self.action_indices,
|
| 407 |
+
modality_keys=self.action_keys,
|
| 408 |
+
)
|
| 409 |
+
language_modality = ModalityConfig(
|
| 410 |
+
delta_indices=self.observation_indices,
|
| 411 |
+
modality_keys=self.language_keys,
|
| 412 |
+
)
|
| 413 |
+
modality_configs = {
|
| 414 |
+
"video": video_modality,
|
| 415 |
+
"state": state_modality,
|
| 416 |
+
"action": action_modality,
|
| 417 |
+
"language": language_modality,
|
| 418 |
+
}
|
| 419 |
+
return modality_configs
|
| 420 |
+
|
| 421 |
+
def transform(self):
|
| 422 |
+
transforms = [
|
| 423 |
+
# state transforms
|
| 424 |
+
StateActionToTensor(apply_to=self.state_keys),
|
| 425 |
+
StateActionTransform(
|
| 426 |
+
apply_to=self.state_keys,
|
| 427 |
+
normalization_modes={
|
| 428 |
+
"state.eef_position": "min_max",
|
| 429 |
+
"state.eef_rotation": "min_max",
|
| 430 |
+
},
|
| 431 |
+
),
|
| 432 |
+
# action transforms
|
| 433 |
+
StateActionToTensor(apply_to=self.action_keys),
|
| 434 |
+
StateActionTransform(
|
| 435 |
+
apply_to=self.action_keys,
|
| 436 |
+
normalization_modes={
|
| 437 |
+
"action.delta_eef_position": "min_max",
|
| 438 |
+
"action.delta_eef_rotation": "min_max",
|
| 439 |
+
"action.gripper_close": "binary",
|
| 440 |
+
},
|
| 441 |
+
),
|
| 442 |
+
]
|
| 443 |
+
|
| 444 |
+
return ComposedModalityTransform(transforms=transforms)
|
| 445 |
+
|
| 446 |
+
###########################################################################################
|
| 447 |
+
|
| 448 |
+
class Libero4in1DataConfig:
|
| 449 |
+
video_keys = [
|
| 450 |
+
"video.primary_image",
|
| 451 |
+
"video.wrist_image",
|
| 452 |
+
]
|
| 453 |
+
|
| 454 |
+
state_keys = [
|
| 455 |
+
"state.x",
|
| 456 |
+
"state.y",
|
| 457 |
+
"state.z",
|
| 458 |
+
"state.roll",
|
| 459 |
+
"state.pitch",
|
| 460 |
+
"state.yaw",
|
| 461 |
+
"state.pad",
|
| 462 |
+
"state.gripper",
|
| 463 |
+
]
|
| 464 |
+
action_keys = [
|
| 465 |
+
"action.x",
|
| 466 |
+
"action.y",
|
| 467 |
+
"action.z",
|
| 468 |
+
"action.roll",
|
| 469 |
+
"action.pitch",
|
| 470 |
+
"action.yaw",
|
| 471 |
+
"action.gripper",
|
| 472 |
+
]
|
| 473 |
+
|
| 474 |
+
language_keys = ["annotation.human.action.task_description"]
|
| 475 |
+
|
| 476 |
+
observation_indices = [0]
|
| 477 |
+
action_indices = list(range(8))
|
| 478 |
+
|
| 479 |
+
|
| 480 |
+
def modality_config(self):
|
| 481 |
+
video_modality = ModalityConfig(
|
| 482 |
+
delta_indices=self.observation_indices,
|
| 483 |
+
modality_keys=self.video_keys,
|
| 484 |
+
)
|
| 485 |
+
state_modality = ModalityConfig(
|
| 486 |
+
delta_indices=self.observation_indices,
|
| 487 |
+
modality_keys=self.state_keys,
|
| 488 |
+
)
|
| 489 |
+
action_modality = ModalityConfig(
|
| 490 |
+
delta_indices=self.action_indices,
|
| 491 |
+
modality_keys=self.action_keys,
|
| 492 |
+
)
|
| 493 |
+
language_modality = ModalityConfig(
|
| 494 |
+
delta_indices=self.observation_indices,
|
| 495 |
+
modality_keys=self.language_keys,
|
| 496 |
+
)
|
| 497 |
+
modality_configs = {
|
| 498 |
+
"video": video_modality,
|
| 499 |
+
"state": state_modality,
|
| 500 |
+
"action": action_modality,
|
| 501 |
+
"language": language_modality,
|
| 502 |
+
}
|
| 503 |
+
return modality_configs
|
| 504 |
+
|
| 505 |
+
def transform(self):
|
| 506 |
+
transforms = [
|
| 507 |
+
# action transforms
|
| 508 |
+
StateActionToTensor(apply_to=self.action_keys),
|
| 509 |
+
StateActionTransform(
|
| 510 |
+
apply_to=self.action_keys,
|
| 511 |
+
normalization_modes={
|
| 512 |
+
"action.x": "min_max",
|
| 513 |
+
"action.y": "min_max",
|
| 514 |
+
"action.z": "min_max",
|
| 515 |
+
"action.roll": "min_max",
|
| 516 |
+
"action.pitch": "min_max",
|
| 517 |
+
"action.yaw": "min_max",
|
| 518 |
+
},
|
| 519 |
+
),
|
| 520 |
+
]
|
| 521 |
+
|
| 522 |
+
return ComposedModalityTransform(transforms=transforms)
|
| 523 |
+
|
| 524 |
+
###########################################################################################
|
| 525 |
+
|
| 526 |
+
|
| 527 |
+
class SingleFrankaRobotiqDeltaJointsDataConfig:
|
| 528 |
+
video_keys = [
|
| 529 |
+
"video.base_view",
|
| 530 |
+
"video.ego_view",
|
| 531 |
+
]
|
| 532 |
+
state_keys = [
|
| 533 |
+
"state.joints",
|
| 534 |
+
]
|
| 535 |
+
action_keys = [
|
| 536 |
+
"action.delta_joints",
|
| 537 |
+
"action.gripper_close",
|
| 538 |
+
]
|
| 539 |
+
|
| 540 |
+
language_keys = ["annotation.human.action.task_description"]
|
| 541 |
+
observation_indices = [0]
|
| 542 |
+
action_indices = list(range(16))
|
| 543 |
+
|
| 544 |
+
def modality_config(self):
|
| 545 |
+
video_modality = ModalityConfig(
|
| 546 |
+
delta_indices=self.observation_indices,
|
| 547 |
+
modality_keys=self.video_keys,
|
| 548 |
+
)
|
| 549 |
+
state_modality = ModalityConfig(
|
| 550 |
+
delta_indices=self.observation_indices,
|
| 551 |
+
modality_keys=self.state_keys,
|
| 552 |
+
)
|
| 553 |
+
action_modality = ModalityConfig(
|
| 554 |
+
delta_indices=self.action_indices,
|
| 555 |
+
modality_keys=self.action_keys,
|
| 556 |
+
)
|
| 557 |
+
language_modality = ModalityConfig(
|
| 558 |
+
delta_indices=self.observation_indices,
|
| 559 |
+
modality_keys=self.language_keys,
|
| 560 |
+
)
|
| 561 |
+
modality_configs = {
|
| 562 |
+
"video": video_modality,
|
| 563 |
+
"state": state_modality,
|
| 564 |
+
"action": action_modality,
|
| 565 |
+
"language": language_modality,
|
| 566 |
+
}
|
| 567 |
+
return modality_configs
|
| 568 |
+
|
| 569 |
+
def transform(self):
|
| 570 |
+
transforms = [
|
| 571 |
+
# state transforms
|
| 572 |
+
StateActionToTensor(apply_to=self.state_keys),
|
| 573 |
+
StateActionTransform(
|
| 574 |
+
apply_to=self.state_keys,
|
| 575 |
+
normalization_modes={
|
| 576 |
+
"state.joints": "min_max",
|
| 577 |
+
},
|
| 578 |
+
),
|
| 579 |
+
# action transforms
|
| 580 |
+
StateActionToTensor(apply_to=self.action_keys),
|
| 581 |
+
StateActionTransform(
|
| 582 |
+
apply_to=self.action_keys,
|
| 583 |
+
normalization_modes={
|
| 584 |
+
"action.delta_joints": "min_max",
|
| 585 |
+
"action.gripper_close": "binary",
|
| 586 |
+
},
|
| 587 |
+
),
|
| 588 |
+
]
|
| 589 |
+
|
| 590 |
+
return ComposedModalityTransform(transforms=transforms)
|
| 591 |
+
|
| 592 |
+
class HumanoidDataConfig:
|
| 593 |
+
video_keys = [
|
| 594 |
+
"video.rs_view",
|
| 595 |
+
]
|
| 596 |
+
state_keys = [
|
| 597 |
+
"state.left_hand",
|
| 598 |
+
"state.right_hand",
|
| 599 |
+
"state.left_arm",
|
| 600 |
+
"state.right_arm",
|
| 601 |
+
"state.rpy",
|
| 602 |
+
"state.height",
|
| 603 |
+
]
|
| 604 |
+
action_keys = [
|
| 605 |
+
"action.left_hand",
|
| 606 |
+
"action.right_hand",
|
| 607 |
+
"action.left_arm",
|
| 608 |
+
"action.right_arm",
|
| 609 |
+
"action.rpy",
|
| 610 |
+
"action.height",
|
| 611 |
+
"action.torso_vx",
|
| 612 |
+
"action.torso_vy",
|
| 613 |
+
"action.torso_vyaw",
|
| 614 |
+
"action.target_yaw",
|
| 615 |
+
]
|
| 616 |
+
|
| 617 |
+
language_keys = ["annotation.human.task_description"]
|
| 618 |
+
observation_indices = [0]
|
| 619 |
+
action_indices = list(range(16)) # list(range(30))
|
| 620 |
+
|
| 621 |
+
def modality_config(self):
|
| 622 |
+
video_modality = ModalityConfig(
|
| 623 |
+
delta_indices=self.observation_indices,
|
| 624 |
+
modality_keys=self.video_keys,
|
| 625 |
+
)
|
| 626 |
+
state_modality = ModalityConfig(
|
| 627 |
+
delta_indices=self.observation_indices,
|
| 628 |
+
modality_keys=self.state_keys,
|
| 629 |
+
)
|
| 630 |
+
action_modality = ModalityConfig(
|
| 631 |
+
delta_indices=self.action_indices,
|
| 632 |
+
modality_keys=self.action_keys,
|
| 633 |
+
)
|
| 634 |
+
language_modality = ModalityConfig(
|
| 635 |
+
delta_indices=self.observation_indices,
|
| 636 |
+
modality_keys=self.language_keys,
|
| 637 |
+
)
|
| 638 |
+
modality_configs = {
|
| 639 |
+
"video": video_modality,
|
| 640 |
+
"state": state_modality,
|
| 641 |
+
"action": action_modality,
|
| 642 |
+
"language": language_modality,
|
| 643 |
+
}
|
| 644 |
+
return modality_configs
|
| 645 |
+
|
| 646 |
+
def transform(self):
|
| 647 |
+
transforms = [
|
| 648 |
+
# state transforms
|
| 649 |
+
StateActionToTensor(apply_to=self.state_keys),
|
| 650 |
+
StateActionTransform(
|
| 651 |
+
apply_to=self.state_keys,
|
| 652 |
+
normalization_modes={
|
| 653 |
+
"state.left_hand": "min_max",
|
| 654 |
+
"state.right_hand": "min_max",
|
| 655 |
+
"state.left_arm": "min_max",
|
| 656 |
+
"state.right_arm": "min_max",
|
| 657 |
+
"state.rpy": "min_max",
|
| 658 |
+
"state.height": "min_max",
|
| 659 |
+
},
|
| 660 |
+
),
|
| 661 |
+
# action transforms
|
| 662 |
+
StateActionToTensor(apply_to=self.action_keys),
|
| 663 |
+
StateActionTransform(
|
| 664 |
+
apply_to=self.action_keys,
|
| 665 |
+
normalization_modes={
|
| 666 |
+
"action.left_hand": "min_max",
|
| 667 |
+
"action.right_hand": "min_max",
|
| 668 |
+
"action.left_arm": "min_max",
|
| 669 |
+
"action.right_arm": "min_max",
|
| 670 |
+
"action.rpy": "min_max",
|
| 671 |
+
"action.height": "min_max",
|
| 672 |
+
"action.torso_vx": "min_max",
|
| 673 |
+
"action.torso_vy": "min_max",
|
| 674 |
+
"action.torso_vyaw": "min_max",
|
| 675 |
+
"action.target_yaw": "min_max",
|
| 676 |
+
},
|
| 677 |
+
),
|
| 678 |
+
]
|
| 679 |
+
|
| 680 |
+
return ComposedModalityTransform(transforms=transforms)
|
| 681 |
+
|
| 682 |
+
|
| 683 |
+
class HumanoidSimDataConfig:
|
| 684 |
+
video_keys = ["video.rs_view"]
|
| 685 |
+
state_keys = [
|
| 686 |
+
"state.left_hand",
|
| 687 |
+
"state.right_hand",
|
| 688 |
+
"state.left_arm",
|
| 689 |
+
"state.right_arm",
|
| 690 |
+
"state.waist",
|
| 691 |
+
]
|
| 692 |
+
action_keys = [
|
| 693 |
+
"action.left_hand",
|
| 694 |
+
"action.right_hand",
|
| 695 |
+
"action.left_arm",
|
| 696 |
+
"action.right_arm",
|
| 697 |
+
"action.waist",
|
| 698 |
+
]
|
| 699 |
+
language_keys = ["annotation.human.task_description"]
|
| 700 |
+
observation_indices = [0]
|
| 701 |
+
action_indices = list(range(16))
|
| 702 |
+
|
| 703 |
+
def modality_config(self):
|
| 704 |
+
video_modality = ModalityConfig(
|
| 705 |
+
delta_indices=self.observation_indices,
|
| 706 |
+
modality_keys=self.video_keys,
|
| 707 |
+
)
|
| 708 |
+
state_modality = ModalityConfig(
|
| 709 |
+
delta_indices=self.observation_indices,
|
| 710 |
+
modality_keys=self.state_keys,
|
| 711 |
+
)
|
| 712 |
+
action_modality = ModalityConfig(
|
| 713 |
+
delta_indices=self.action_indices,
|
| 714 |
+
modality_keys=self.action_keys,
|
| 715 |
+
)
|
| 716 |
+
language_modality = ModalityConfig(
|
| 717 |
+
delta_indices=self.observation_indices,
|
| 718 |
+
modality_keys=self.language_keys,
|
| 719 |
+
)
|
| 720 |
+
return {
|
| 721 |
+
"video": video_modality,
|
| 722 |
+
"state": state_modality,
|
| 723 |
+
"action": action_modality,
|
| 724 |
+
"language": language_modality,
|
| 725 |
+
}
|
| 726 |
+
|
| 727 |
+
def transform(self):
|
| 728 |
+
transforms = [
|
| 729 |
+
VideoToTensor(apply_to=self.video_keys),
|
| 730 |
+
VideoCrop(apply_to=self.video_keys, scale=0.95),
|
| 731 |
+
VideoResize(apply_to=self.video_keys, height=224, width=224, interpolation="linear"),
|
| 732 |
+
VideoColorJitter(
|
| 733 |
+
apply_to=self.video_keys,
|
| 734 |
+
brightness=0.3,
|
| 735 |
+
contrast=0.4,
|
| 736 |
+
saturation=0.5,
|
| 737 |
+
hue=0.08,
|
| 738 |
+
),
|
| 739 |
+
VideoToNumpy(apply_to=self.video_keys),
|
| 740 |
+
StateActionToTensor(apply_to=self.state_keys),
|
| 741 |
+
StateActionTransform(
|
| 742 |
+
apply_to=self.state_keys,
|
| 743 |
+
normalization_modes={key: "min_max" for key in self.state_keys},
|
| 744 |
+
),
|
| 745 |
+
StateActionToTensor(apply_to=self.action_keys),
|
| 746 |
+
StateActionTransform(
|
| 747 |
+
apply_to=self.action_keys,
|
| 748 |
+
normalization_modes={key: "min_max" for key in self.action_keys},
|
| 749 |
+
),
|
| 750 |
+
ConcatTransform(
|
| 751 |
+
video_concat_order=self.video_keys,
|
| 752 |
+
state_concat_order=self.state_keys,
|
| 753 |
+
action_concat_order=self.action_keys,
|
| 754 |
+
),
|
| 755 |
+
]
|
| 756 |
+
return ComposedModalityTransform(transforms=transforms)
|
| 757 |
+
###########################################################################################
|
| 758 |
+
|
| 759 |
+
|
| 760 |
+
|
| 761 |
+
ROBOT_TYPE_CONFIG_MAP = {
|
| 762 |
+
"libero_franka": Libero4in1DataConfig(),
|
| 763 |
+
"oxe_droid": OxeDroidDataConfig(),
|
| 764 |
+
"oxe_bridge": OxeBridgeDataConfig(),
|
| 765 |
+
"oxe_rt1": OxeRT1DataConfig(),
|
| 766 |
+
"demo_sim_franka_delta_joints": SingleFrankaRobotiqDeltaJointsDataConfig(),
|
| 767 |
+
"custom_robot_config": SingleFrankaRobotiqDeltaEefDataConfig(),
|
| 768 |
+
"humanoid_config": HumanoidDataConfig(),
|
| 769 |
+
"humanoid_sim_config": HumanoidSimDataConfig(),
|
| 770 |
+
}
|
src/InternVLA-M1/InternVLA/dataloader/gr00t_lerobot/datasets.py
ADDED
|
@@ -0,0 +1,2118 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
| 2 |
+
# SPDX-License-Identifier: Apache-2.0
|
| 3 |
+
#
|
| 4 |
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
| 5 |
+
# you may not use this file except in compliance with the License.
|
| 6 |
+
# You may obtain a copy of the License at
|
| 7 |
+
#
|
| 8 |
+
# http://www.apache.org/licenses/LICENSE-2.0
|
| 9 |
+
#
|
| 10 |
+
# Unless required by applicable law or agreed to in writing, software
|
| 11 |
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
| 12 |
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
| 13 |
+
# See the License for the specific language governing permissions and
|
| 14 |
+
# limitations under the License.
|
| 15 |
+
|
| 16 |
+
|
| 17 |
+
"""
|
| 18 |
+
In this file, we define 3 types of datasets:
|
| 19 |
+
1. LeRobotSingleDataset: a single dataset for a given embodiment tag
|
| 20 |
+
2. LeRobotMixtureDataset: a mixture of datasets for a given list of embodiment tags
|
| 21 |
+
3. CachedLeRobotSingleDataset: a single dataset for a given embodiment tag,
|
| 22 |
+
with caching for the video frames
|
| 23 |
+
|
| 24 |
+
See `scripts/load_dataset.py` for examples on how to use these datasets.
|
| 25 |
+
"""
|
| 26 |
+
|
| 27 |
+
import hashlib
|
| 28 |
+
import json
|
| 29 |
+
from collections import defaultdict
|
| 30 |
+
from pathlib import Path
|
| 31 |
+
from typing import Sequence
|
| 32 |
+
|
| 33 |
+
import numpy as np
|
| 34 |
+
import pandas as pd
|
| 35 |
+
from pydantic import BaseModel, Field, ValidationError
|
| 36 |
+
from torch.utils.data import Dataset
|
| 37 |
+
from tqdm import tqdm
|
| 38 |
+
from PIL import Image
|
| 39 |
+
|
| 40 |
+
from InternVLA.dataloader.gr00t_lerobot.video import get_all_frames, get_frames_by_timestamps
|
| 41 |
+
|
| 42 |
+
from InternVLA.dataloader.gr00t_lerobot.embodiment_tags import EmbodimentTag
|
| 43 |
+
from InternVLA.dataloader.gr00t_lerobot.schema import (
|
| 44 |
+
DatasetMetadata,
|
| 45 |
+
DatasetStatisticalValues,
|
| 46 |
+
LeRobotModalityMetadata,
|
| 47 |
+
LeRobotStateActionMetadata,
|
| 48 |
+
)
|
| 49 |
+
from InternVLA.dataloader.gr00t_lerobot.transform import ComposedModalityTransform
|
| 50 |
+
|
| 51 |
+
from functools import partial
|
| 52 |
+
from typing import Tuple, List
|
| 53 |
+
import pickle
|
| 54 |
+
|
| 55 |
+
LE_ROBOT_MODALITY_FILENAME = "meta/modality.json"
|
| 56 |
+
LE_ROBOT_EPISODE_FILENAME = "meta/episodes.jsonl"
|
| 57 |
+
LE_ROBOT_TASKS_FILENAME = "meta/tasks.jsonl"
|
| 58 |
+
LE_ROBOT_INFO_FILENAME = "meta/info.json"
|
| 59 |
+
LE_ROBOT_STATS_FILENAME = "meta/stats_gr00t.json"
|
| 60 |
+
LE_ROBOT_DATA_FILENAME = "data/*/*.parquet"
|
| 61 |
+
LE_ROBOT_STEPS_FILENAME = "meta/steps.pkl"
|
| 62 |
+
EPSILON = 5e-4
|
| 63 |
+
|
| 64 |
+
def calculate_dataset_statistics(parquet_paths: list[Path]) -> dict:
|
| 65 |
+
"""Calculate the dataset statistics of all columns for a list of parquet files."""
|
| 66 |
+
# Dataset statistics
|
| 67 |
+
all_low_dim_data_list = []
|
| 68 |
+
# Collect all the data
|
| 69 |
+
# parquet_paths = parquet_paths[:3]
|
| 70 |
+
for parquet_path in tqdm(
|
| 71 |
+
sorted(list(parquet_paths)),
|
| 72 |
+
desc="Collecting all parquet files...",
|
| 73 |
+
):
|
| 74 |
+
# Load the parquet file
|
| 75 |
+
parquet_data = pd.read_parquet(parquet_path)
|
| 76 |
+
parquet_data = parquet_data
|
| 77 |
+
all_low_dim_data_list.append(parquet_data)
|
| 78 |
+
all_low_dim_data = pd.concat(all_low_dim_data_list, axis=0)
|
| 79 |
+
# Compute dataset statistics
|
| 80 |
+
dataset_statistics = {}
|
| 81 |
+
for le_modality in all_low_dim_data.columns:
|
| 82 |
+
if le_modality not in ["states", "action"]:
|
| 83 |
+
continue
|
| 84 |
+
if le_modality.startswith("annotation."):
|
| 85 |
+
continue
|
| 86 |
+
print(f"Computing statistics for {le_modality}...")
|
| 87 |
+
np_data = np.vstack(
|
| 88 |
+
[np.asarray(x, dtype=np.float32) for x in all_low_dim_data[le_modality]]
|
| 89 |
+
)
|
| 90 |
+
dataset_statistics[le_modality] = {
|
| 91 |
+
"mean": np.mean(np_data, axis=0).tolist(),
|
| 92 |
+
"std": np.std(np_data, axis=0).tolist(),
|
| 93 |
+
"min": np.min(np_data, axis=0).tolist(),
|
| 94 |
+
"max": np.max(np_data, axis=0).tolist(),
|
| 95 |
+
"q01": np.quantile(np_data, 0.01, axis=0).tolist(),
|
| 96 |
+
"q99": np.quantile(np_data, 0.99, axis=0).tolist(),
|
| 97 |
+
}
|
| 98 |
+
return dataset_statistics
|
| 99 |
+
|
| 100 |
+
|
| 101 |
+
class ModalityConfig(BaseModel):
|
| 102 |
+
"""Configuration for a modality."""
|
| 103 |
+
|
| 104 |
+
delta_indices: list[int]
|
| 105 |
+
"""Delta indices to sample relative to the current index. The returned data will correspond to the original data at a sampled base index + delta indices."""
|
| 106 |
+
modality_keys: list[str]
|
| 107 |
+
"""The keys to load for the modality in the dataset."""
|
| 108 |
+
|
| 109 |
+
|
| 110 |
+
class LeRobotSingleDataset(Dataset):
|
| 111 |
+
"""
|
| 112 |
+
Base dataset class for LeRobot that supports sharding.
|
| 113 |
+
"""
|
| 114 |
+
def __init__(
|
| 115 |
+
self,
|
| 116 |
+
dataset_path: Path | str,
|
| 117 |
+
modality_configs: dict[str, ModalityConfig],
|
| 118 |
+
embodiment_tag: str | EmbodimentTag,
|
| 119 |
+
video_backend: str = "decord",
|
| 120 |
+
video_backend_kwargs: dict | None = None,
|
| 121 |
+
transforms: ComposedModalityTransform | None = None,
|
| 122 |
+
delete_pause_frame: bool = False,
|
| 123 |
+
preload_all=True
|
| 124 |
+
):
|
| 125 |
+
"""
|
| 126 |
+
Initialize the dataset.
|
| 127 |
+
|
| 128 |
+
Args:
|
| 129 |
+
dataset_path (Path | str): The path to the dataset.
|
| 130 |
+
modality_configs (dict[str, ModalityConfig]): The configuration for each modality. The keys are the modality names, and the values are the modality configurations.
|
| 131 |
+
See `ModalityConfig` for more details.
|
| 132 |
+
video_backend (str): Backend for video reading.
|
| 133 |
+
video_backend_kwargs (dict): Keyword arguments for the video backend when initializing the video reader.
|
| 134 |
+
transforms (ComposedModalityTransform): The transforms to apply to the dataset.
|
| 135 |
+
embodiment_tag (EmbodimentTag): Overload the embodiment tag for the dataset. e.g. define it as "new_embodiment"
|
| 136 |
+
"""
|
| 137 |
+
# first check if the path directory exists
|
| 138 |
+
if not Path(dataset_path).exists():
|
| 139 |
+
raise FileNotFoundError(f"Dataset path {dataset_path} does not exist")
|
| 140 |
+
|
| 141 |
+
self.delete_pause_frame = delete_pause_frame
|
| 142 |
+
|
| 143 |
+
self.modality_configs = modality_configs
|
| 144 |
+
self.video_backend = video_backend
|
| 145 |
+
self.video_backend_kwargs = video_backend_kwargs if video_backend_kwargs is not None else {}
|
| 146 |
+
self.transforms = (
|
| 147 |
+
transforms if transforms is not None else ComposedModalityTransform(transforms=[])
|
| 148 |
+
)
|
| 149 |
+
|
| 150 |
+
self._dataset_path = Path(dataset_path)
|
| 151 |
+
self._dataset_name = self._dataset_path.name
|
| 152 |
+
if isinstance(embodiment_tag, EmbodimentTag):
|
| 153 |
+
self.tag = embodiment_tag.value
|
| 154 |
+
else:
|
| 155 |
+
self.tag = embodiment_tag
|
| 156 |
+
|
| 157 |
+
self._metadata = self._get_metadata(EmbodimentTag(self.tag))
|
| 158 |
+
|
| 159 |
+
# LeRobot-specific config
|
| 160 |
+
self._lerobot_modality_meta = self._get_lerobot_modality_meta()
|
| 161 |
+
self._lerobot_info_meta = self._get_lerobot_info_meta()
|
| 162 |
+
self._data_path_pattern = self._get_data_path_pattern()
|
| 163 |
+
self._video_path_pattern = self._get_video_path_pattern()
|
| 164 |
+
self._chunk_size = self._get_chunk_size()
|
| 165 |
+
self._tasks = self._get_tasks()
|
| 166 |
+
self.curr_traj_data = None
|
| 167 |
+
self.curr_traj_id = None
|
| 168 |
+
|
| 169 |
+
self._trajectory_ids, self._trajectory_lengths = self._get_trajectories()
|
| 170 |
+
if preload_all:
|
| 171 |
+
self._preload_all_trajectories()
|
| 172 |
+
self.preload_all = True
|
| 173 |
+
else:
|
| 174 |
+
self.preload_all = False
|
| 175 |
+
self._modality_keys = self._get_modality_keys()
|
| 176 |
+
self._delta_indices = self._get_delta_indices()
|
| 177 |
+
self._all_steps = self._get_all_steps()
|
| 178 |
+
self.set_transforms_metadata(self.metadata)
|
| 179 |
+
self.set_epoch(0)
|
| 180 |
+
|
| 181 |
+
print(f"Initialized dataset {self.dataset_name} with {embodiment_tag}")
|
| 182 |
+
|
| 183 |
+
|
| 184 |
+
# Check if the dataset is valid
|
| 185 |
+
self._check_integrity()
|
| 186 |
+
|
| 187 |
+
@property
|
| 188 |
+
def dataset_path(self) -> Path:
|
| 189 |
+
"""The path to the dataset that contains the METADATA_FILENAME file."""
|
| 190 |
+
return self._dataset_path
|
| 191 |
+
|
| 192 |
+
@property
|
| 193 |
+
def metadata(self) -> DatasetMetadata:
|
| 194 |
+
"""The metadata for the dataset, loaded from metadata.json in the dataset directory"""
|
| 195 |
+
return self._metadata
|
| 196 |
+
|
| 197 |
+
@property
|
| 198 |
+
def trajectory_ids(self) -> np.ndarray:
|
| 199 |
+
"""The trajectory IDs in the dataset, stored as a 1D numpy array of strings."""
|
| 200 |
+
return self._trajectory_ids
|
| 201 |
+
|
| 202 |
+
@property
|
| 203 |
+
def trajectory_lengths(self) -> np.ndarray:
|
| 204 |
+
"""The trajectory lengths in the dataset, stored as a 1D numpy array of integers.
|
| 205 |
+
The order of the lengths is the same as the order of the trajectory IDs.
|
| 206 |
+
"""
|
| 207 |
+
return self._trajectory_lengths
|
| 208 |
+
|
| 209 |
+
@property
|
| 210 |
+
def all_steps(self) -> list[tuple[int, int]]:
|
| 211 |
+
"""The trajectory IDs and base indices for all steps in the dataset.
|
| 212 |
+
Example:
|
| 213 |
+
self.trajectory_ids: [0, 1, 2]
|
| 214 |
+
self.trajectory_lengths: [3, 2, 4]
|
| 215 |
+
return: [
|
| 216 |
+
("traj_0", 0), ("traj_0", 1), ("traj_0", 2),
|
| 217 |
+
("traj_1", 0), ("traj_1", 1),
|
| 218 |
+
("traj_2", 0), ("traj_2", 1), ("traj_2", 2), ("traj_2", 3)
|
| 219 |
+
]
|
| 220 |
+
"""
|
| 221 |
+
return self._all_steps
|
| 222 |
+
|
| 223 |
+
@property
|
| 224 |
+
def modality_keys(self) -> dict:
|
| 225 |
+
"""The modality keys for the dataset. The keys are the modality names, and the values are the keys for each modality.
|
| 226 |
+
|
| 227 |
+
Example: {
|
| 228 |
+
"video": ["video.image_side_0", "video.image_side_1"],
|
| 229 |
+
"state": ["state.eef_position", "state.eef_rotation"],
|
| 230 |
+
"action": ["action.eef_position", "action.eef_rotation"],
|
| 231 |
+
"language": ["language.human.task"],
|
| 232 |
+
"timestamp": ["timestamp"],
|
| 233 |
+
"reward": ["reward"],
|
| 234 |
+
}
|
| 235 |
+
"""
|
| 236 |
+
return self._modality_keys
|
| 237 |
+
|
| 238 |
+
@property
|
| 239 |
+
def delta_indices(self) -> dict[str, np.ndarray]:
|
| 240 |
+
"""The delta indices for the dataset. The keys are the modality.key, and the values are the delta indices for each modality.key."""
|
| 241 |
+
return self._delta_indices
|
| 242 |
+
|
| 243 |
+
@property
|
| 244 |
+
def dataset_name(self) -> str:
|
| 245 |
+
"""The name of the dataset."""
|
| 246 |
+
return self._dataset_name
|
| 247 |
+
|
| 248 |
+
@property
|
| 249 |
+
def lerobot_modality_meta(self) -> LeRobotModalityMetadata:
|
| 250 |
+
"""The metadata for the LeRobot dataset."""
|
| 251 |
+
return self._lerobot_modality_meta
|
| 252 |
+
|
| 253 |
+
@property
|
| 254 |
+
def lerobot_info_meta(self) -> dict:
|
| 255 |
+
"""The metadata for the LeRobot dataset."""
|
| 256 |
+
return self._lerobot_info_meta
|
| 257 |
+
|
| 258 |
+
@property
|
| 259 |
+
def data_path_pattern(self) -> str:
|
| 260 |
+
"""The path pattern for the LeRobot dataset."""
|
| 261 |
+
return self._data_path_pattern
|
| 262 |
+
|
| 263 |
+
@property
|
| 264 |
+
def video_path_pattern(self) -> str:
|
| 265 |
+
"""The path pattern for the LeRobot dataset."""
|
| 266 |
+
return self._video_path_pattern
|
| 267 |
+
|
| 268 |
+
@property
|
| 269 |
+
def chunk_size(self) -> int:
|
| 270 |
+
"""The chunk size for the LeRobot dataset."""
|
| 271 |
+
return self._chunk_size
|
| 272 |
+
|
| 273 |
+
@property
|
| 274 |
+
def tasks(self) -> pd.DataFrame:
|
| 275 |
+
"""The tasks for the dataset."""
|
| 276 |
+
return self._tasks
|
| 277 |
+
|
| 278 |
+
def _get_metadata(self, embodiment_tag: EmbodimentTag) -> DatasetMetadata:
|
| 279 |
+
"""Get the metadata for the dataset.
|
| 280 |
+
|
| 281 |
+
Returns:
|
| 282 |
+
dict: The metadata for the dataset.
|
| 283 |
+
"""
|
| 284 |
+
|
| 285 |
+
# 1. Modality metadata
|
| 286 |
+
modality_meta_path = self.dataset_path / LE_ROBOT_MODALITY_FILENAME
|
| 287 |
+
assert (
|
| 288 |
+
modality_meta_path.exists()
|
| 289 |
+
), f"Please provide a {LE_ROBOT_MODALITY_FILENAME} file in {self.dataset_path}"
|
| 290 |
+
# 1.1. State and action modalities
|
| 291 |
+
simplified_modality_meta: dict[str, dict] = {}
|
| 292 |
+
with open(modality_meta_path, "r") as f:
|
| 293 |
+
le_modality_meta = LeRobotModalityMetadata.model_validate(json.load(f))
|
| 294 |
+
for modality in ["state", "action"]:
|
| 295 |
+
simplified_modality_meta[modality] = {}
|
| 296 |
+
le_state_action_meta: dict[str, LeRobotStateActionMetadata] = getattr(
|
| 297 |
+
le_modality_meta, modality
|
| 298 |
+
)
|
| 299 |
+
for subkey in le_state_action_meta:
|
| 300 |
+
state_action_dtype = np.dtype(le_state_action_meta[subkey].dtype)
|
| 301 |
+
if np.issubdtype(state_action_dtype, np.floating):
|
| 302 |
+
continuous = True
|
| 303 |
+
else:
|
| 304 |
+
continuous = False
|
| 305 |
+
simplified_modality_meta[modality][subkey] = {
|
| 306 |
+
"absolute": le_state_action_meta[subkey].absolute,
|
| 307 |
+
"rotation_type": le_state_action_meta[subkey].rotation_type,
|
| 308 |
+
"shape": [
|
| 309 |
+
le_state_action_meta[subkey].end - le_state_action_meta[subkey].start
|
| 310 |
+
],
|
| 311 |
+
"continuous": continuous,
|
| 312 |
+
}
|
| 313 |
+
|
| 314 |
+
# 1.2. Video modalities
|
| 315 |
+
le_info_path = self.dataset_path / LE_ROBOT_INFO_FILENAME
|
| 316 |
+
assert (
|
| 317 |
+
le_info_path.exists()
|
| 318 |
+
), f"Please provide a {LE_ROBOT_INFO_FILENAME} file in {self.dataset_path}"
|
| 319 |
+
with open(le_info_path, "r") as f:
|
| 320 |
+
le_info = json.load(f)
|
| 321 |
+
simplified_modality_meta["video"] = {}
|
| 322 |
+
for new_key in le_modality_meta.video:
|
| 323 |
+
original_key = le_modality_meta.video[new_key].original_key
|
| 324 |
+
if original_key is None:
|
| 325 |
+
original_key = new_key
|
| 326 |
+
le_video_meta = le_info["features"][original_key]
|
| 327 |
+
height = le_video_meta["shape"][le_video_meta["names"].index("height")]
|
| 328 |
+
width = le_video_meta["shape"][le_video_meta["names"].index("width")]
|
| 329 |
+
# NOTE(FH): different lerobot dataset versions have different keys for the number of channels and fps
|
| 330 |
+
try:
|
| 331 |
+
channels = le_video_meta["shape"][le_video_meta["names"].index("channel")]
|
| 332 |
+
fps = le_video_meta["video_info"]["video.fps"]
|
| 333 |
+
except (ValueError, KeyError):
|
| 334 |
+
# channels = le_video_meta["shape"][le_video_meta["names"].index("channels")]
|
| 335 |
+
channels = le_video_meta["info"]["video.channels"]
|
| 336 |
+
fps = le_video_meta["info"]["video.fps"]
|
| 337 |
+
simplified_modality_meta["video"][new_key] = {
|
| 338 |
+
"resolution": [width, height],
|
| 339 |
+
"channels": channels,
|
| 340 |
+
"fps": fps,
|
| 341 |
+
}
|
| 342 |
+
|
| 343 |
+
# 2. Dataset statistics
|
| 344 |
+
stats_path = self.dataset_path / LE_ROBOT_STATS_FILENAME
|
| 345 |
+
try:
|
| 346 |
+
with open(stats_path, "r") as f:
|
| 347 |
+
le_statistics = json.load(f)
|
| 348 |
+
for stat in le_statistics.values():
|
| 349 |
+
DatasetStatisticalValues.model_validate(stat)
|
| 350 |
+
except (FileNotFoundError, ValidationError) as e:
|
| 351 |
+
print(f"Failed to load dataset statistics: {e}")
|
| 352 |
+
print(f"Calculating dataset statistics for {self.dataset_name}")
|
| 353 |
+
# Get all parquet files in the dataset paths
|
| 354 |
+
parquet_files = list((self.dataset_path).glob(LE_ROBOT_DATA_FILENAME))
|
| 355 |
+
le_statistics = calculate_dataset_statistics(parquet_files)
|
| 356 |
+
with open(stats_path, "w") as f:
|
| 357 |
+
json.dump(le_statistics, f, indent=4)
|
| 358 |
+
dataset_statistics = {}
|
| 359 |
+
for our_modality in ["state", "action"]:
|
| 360 |
+
dataset_statistics[our_modality] = {}
|
| 361 |
+
for subkey in simplified_modality_meta[our_modality]:
|
| 362 |
+
dataset_statistics[our_modality][subkey] = {}
|
| 363 |
+
state_action_meta = le_modality_meta.get_key_meta(f"{our_modality}.{subkey}")
|
| 364 |
+
assert isinstance(state_action_meta, LeRobotStateActionMetadata)
|
| 365 |
+
le_modality = state_action_meta.original_key
|
| 366 |
+
for stat_name in le_statistics[le_modality]:
|
| 367 |
+
indices = np.arange(
|
| 368 |
+
state_action_meta.start,
|
| 369 |
+
state_action_meta.end,
|
| 370 |
+
)
|
| 371 |
+
stat = np.array(le_statistics[le_modality][stat_name])
|
| 372 |
+
dataset_statistics[our_modality][subkey][stat_name] = stat[indices].tolist()
|
| 373 |
+
|
| 374 |
+
# 3. Full dataset metadata
|
| 375 |
+
metadata = DatasetMetadata(
|
| 376 |
+
statistics=dataset_statistics, # type: ignore
|
| 377 |
+
modalities=simplified_modality_meta, # type: ignore
|
| 378 |
+
embodiment_tag=embodiment_tag,
|
| 379 |
+
)
|
| 380 |
+
|
| 381 |
+
return metadata
|
| 382 |
+
|
| 383 |
+
def _get_trajectories(self) -> tuple[np.ndarray, np.ndarray]:
|
| 384 |
+
"""Get the trajectories in the dataset."""
|
| 385 |
+
# Get trajectory lengths, IDs, and whitelist from dataset metadata
|
| 386 |
+
episode_path = self.dataset_path / LE_ROBOT_EPISODE_FILENAME
|
| 387 |
+
with open(episode_path, "r") as f:
|
| 388 |
+
episode_metadata = [json.loads(line) for line in f]
|
| 389 |
+
trajectory_ids = []
|
| 390 |
+
trajectory_lengths = []
|
| 391 |
+
for episode in episode_metadata:
|
| 392 |
+
trajectory_ids.append(episode["episode_index"])
|
| 393 |
+
trajectory_lengths.append(episode["length"])
|
| 394 |
+
return np.array(trajectory_ids), np.array(trajectory_lengths)
|
| 395 |
+
|
| 396 |
+
def _get_all_steps(self) -> list[tuple[int, int]]:
|
| 397 |
+
"""Get the trajectory IDs and base indices for all steps in the dataset.
|
| 398 |
+
|
| 399 |
+
Returns:
|
| 400 |
+
list[tuple[str, int]]: A list of (trajectory_id, base_index) tuples.
|
| 401 |
+
"""
|
| 402 |
+
# Create a hash key based on configuration to ensure cache validity
|
| 403 |
+
config_key = self._get_steps_config_key()
|
| 404 |
+
|
| 405 |
+
# Create a unique filename based on config_key
|
| 406 |
+
steps_filename = f"steps_{config_key}.pkl"
|
| 407 |
+
# @DUG
|
| 408 |
+
# fast get static steps @fangjing --> don't use hash to dynamic sample
|
| 409 |
+
steps_filename = "steps_7799a3080fbd.pkl"
|
| 410 |
+
|
| 411 |
+
steps_path = self.dataset_path / "meta" / steps_filename
|
| 412 |
+
|
| 413 |
+
# Try to load cached steps first
|
| 414 |
+
try:
|
| 415 |
+
if steps_path.exists():
|
| 416 |
+
with open(steps_path, "rb") as f:
|
| 417 |
+
cached_data = pickle.load(f)
|
| 418 |
+
return cached_data["steps"]
|
| 419 |
+
# # Verify the cached data matches current configuration
|
| 420 |
+
# if cached_data.get("config_key") == config_key:
|
| 421 |
+
# print(f"Loading cached steps from {steps_path}")
|
| 422 |
+
# return cached_data["steps"]
|
| 423 |
+
# else:
|
| 424 |
+
# print("Cached steps configuration mismatch, recomputing...")
|
| 425 |
+
except (FileNotFoundError, pickle.PickleError, KeyError) as e:
|
| 426 |
+
print(f"Failed to load cached steps: {e}")
|
| 427 |
+
print("Computing steps from scratch...")
|
| 428 |
+
|
| 429 |
+
# Compute steps using single process
|
| 430 |
+
all_steps = self._get_all_steps_single_process()
|
| 431 |
+
|
| 432 |
+
# Cache the computed steps with unique filename
|
| 433 |
+
try:
|
| 434 |
+
cache_data = {
|
| 435 |
+
"config_key": config_key,
|
| 436 |
+
"steps": all_steps,
|
| 437 |
+
"num_trajectories": len(self.trajectory_ids),
|
| 438 |
+
"total_steps": len(all_steps),
|
| 439 |
+
"computed_timestamp": pd.Timestamp.now().isoformat(),
|
| 440 |
+
"delete_pause_frame": self.delete_pause_frame,
|
| 441 |
+
}
|
| 442 |
+
|
| 443 |
+
# Ensure the meta directory exists
|
| 444 |
+
steps_path.parent.mkdir(parents=True, exist_ok=True)
|
| 445 |
+
|
| 446 |
+
with open(steps_path, "wb") as f:
|
| 447 |
+
pickle.dump(cache_data, f, protocol=pickle.HIGHEST_PROTOCOL)
|
| 448 |
+
print(f"Cached steps saved to {steps_path}")
|
| 449 |
+
except Exception as e:
|
| 450 |
+
print(f"Failed to cache steps: {e}")
|
| 451 |
+
|
| 452 |
+
return all_steps
|
| 453 |
+
|
| 454 |
+
def _get_steps_config_key(self) -> str:
|
| 455 |
+
"""Generate a configuration key for steps caching."""
|
| 456 |
+
config_dict = {
|
| 457 |
+
"delete_pause_frame": self.delete_pause_frame,
|
| 458 |
+
"dataset_name": self.dataset_name,
|
| 459 |
+
}
|
| 460 |
+
# Create a hash of the configuration
|
| 461 |
+
config_str = str(sorted(config_dict.items()))
|
| 462 |
+
return hashlib.md5(config_str.encode()).hexdigest()[:12] #
|
| 463 |
+
|
| 464 |
+
|
| 465 |
+
def _get_all_steps_single_process(self) -> list[tuple[int, int]]:
|
| 466 |
+
"""Original single-process implementation as fallback."""
|
| 467 |
+
all_steps: list[tuple[int, int]] = []
|
| 468 |
+
skipped_trajectories = 0
|
| 469 |
+
processed_trajectories = 0
|
| 470 |
+
|
| 471 |
+
# Check if language modality is configured
|
| 472 |
+
has_language_modality = 'language' in self.modality_keys and len(self.modality_keys['language']) > 0
|
| 473 |
+
|
| 474 |
+
for trajectory_id, trajectory_length in tqdm(zip(self.trajectory_ids, self.trajectory_lengths), total=len(self.trajectory_ids), desc="Getting All Step"):
|
| 475 |
+
data = self.get_trajectory_data(trajectory_id)
|
| 476 |
+
trajectory_skipped = False
|
| 477 |
+
|
| 478 |
+
# Check if trajectory has valid language instruction (if language modality is configured)
|
| 479 |
+
if has_language_modality:
|
| 480 |
+
self.curr_traj_data = data # Set current trajectory data for get_language to work
|
| 481 |
+
try:
|
| 482 |
+
language_instruction = self.get_language(trajectory_id, self.modality_keys['language'][0], 0)
|
| 483 |
+
print(language_instruction)
|
| 484 |
+
if not language_instruction or language_instruction[0] == "":
|
| 485 |
+
print(f"Skipping trajectory {trajectory_id} due to empty language instruction")
|
| 486 |
+
skipped_trajectories += 1
|
| 487 |
+
trajectory_skipped = True
|
| 488 |
+
continue
|
| 489 |
+
except Exception as e:
|
| 490 |
+
print(f"Skipping trajectory {trajectory_id} due to language retrieval error: {e}")
|
| 491 |
+
skipped_trajectories += 1
|
| 492 |
+
trajectory_skipped = True
|
| 493 |
+
continue
|
| 494 |
+
|
| 495 |
+
if not trajectory_skipped:
|
| 496 |
+
processed_trajectories += 1
|
| 497 |
+
|
| 498 |
+
self.delete_pause_frame = False # FIXME
|
| 499 |
+
if self.delete_pause_frame:
|
| 500 |
+
# Get position and gripper fields based on available columns
|
| 501 |
+
delta_position_values, gripper_values = self._get_position_and_gripper_values(data)
|
| 502 |
+
previous_gripper = gripper_values[0]
|
| 503 |
+
for base_index in range(trajectory_length):
|
| 504 |
+
if base_index >= len(delta_position_values) or base_index >= len(gripper_values):
|
| 505 |
+
break
|
| 506 |
+
|
| 507 |
+
# Check for translation change using the detected position fields
|
| 508 |
+
has_translation_change = np.any(np.abs(delta_position_values[base_index]) > EPSILON)
|
| 509 |
+
has_gripper_change = gripper_values[base_index] != (previous_gripper if base_index == 0 else gripper_values[base_index-1])
|
| 510 |
+
|
| 511 |
+
if has_translation_change or has_gripper_change:
|
| 512 |
+
all_steps.append((trajectory_id, base_index))
|
| 513 |
+
else:
|
| 514 |
+
for base_index in range(trajectory_length):
|
| 515 |
+
all_steps.append((trajectory_id, base_index))
|
| 516 |
+
|
| 517 |
+
# Print summary statistics
|
| 518 |
+
print(f"Single-process summary: Processed {processed_trajectories} trajectories, skipped {skipped_trajectories} empty trajectories")
|
| 519 |
+
print(f"Total steps: {len(all_steps)} from {len(self.trajectory_ids)} trajectories")
|
| 520 |
+
|
| 521 |
+
return all_steps
|
| 522 |
+
|
| 523 |
+
def _get_position_and_gripper_values(self, data: pd.DataFrame) -> tuple[list, list]:
|
| 524 |
+
"""Get position and gripper values based on available columns in the dataset."""
|
| 525 |
+
# Get action keys from modality_keys
|
| 526 |
+
action_keys = self.modality_keys.get('action', [])
|
| 527 |
+
|
| 528 |
+
# Extract position data
|
| 529 |
+
delta_position_values = None
|
| 530 |
+
position_candidates = ['delta_eef_position']
|
| 531 |
+
coordinate_candidates = ['x', 'y', 'z']
|
| 532 |
+
|
| 533 |
+
# First try combined position fields
|
| 534 |
+
for pos_key in position_candidates:
|
| 535 |
+
full_key = f"action.{pos_key}"
|
| 536 |
+
if full_key in action_keys:
|
| 537 |
+
try:
|
| 538 |
+
# Get the lerobot key for this modality
|
| 539 |
+
le_action_cfg = self.lerobot_modality_meta.action
|
| 540 |
+
subkey = pos_key
|
| 541 |
+
if subkey in le_action_cfg:
|
| 542 |
+
le_key = le_action_cfg[subkey].original_key or subkey
|
| 543 |
+
if le_key in data.columns:
|
| 544 |
+
data_array = np.stack(data[le_key])
|
| 545 |
+
le_indices = np.arange(le_action_cfg[subkey].start, le_action_cfg[subkey].end)
|
| 546 |
+
filtered_data = data_array[:, le_indices]
|
| 547 |
+
delta_position_values = filtered_data.tolist()
|
| 548 |
+
break
|
| 549 |
+
except Exception:
|
| 550 |
+
continue
|
| 551 |
+
|
| 552 |
+
# If combined fields not found, try individual x,y,z coordinates
|
| 553 |
+
if delta_position_values is None:
|
| 554 |
+
x_data, y_data, z_data = None, None, None
|
| 555 |
+
for coord in coordinate_candidates:
|
| 556 |
+
full_key = f"action.{coord}"
|
| 557 |
+
if full_key in action_keys:
|
| 558 |
+
try:
|
| 559 |
+
le_action_cfg = self.lerobot_modality_meta.action
|
| 560 |
+
if coord in le_action_cfg:
|
| 561 |
+
le_key = le_action_cfg[coord].original_key or coord
|
| 562 |
+
if le_key in data.columns:
|
| 563 |
+
data_array = np.stack(data[le_key])
|
| 564 |
+
le_indices = np.arange(le_action_cfg[coord].start, le_action_cfg[coord].end)
|
| 565 |
+
coord_data = data_array[:, le_indices].flatten()
|
| 566 |
+
if coord == 'x':
|
| 567 |
+
x_data = coord_data
|
| 568 |
+
elif coord == 'y':
|
| 569 |
+
y_data = coord_data
|
| 570 |
+
elif coord == 'z':
|
| 571 |
+
z_data = coord_data
|
| 572 |
+
except Exception:
|
| 573 |
+
continue
|
| 574 |
+
|
| 575 |
+
if x_data is not None and y_data is not None and z_data is not None:
|
| 576 |
+
delta_position_values = np.column_stack((x_data, y_data, z_data)).tolist()
|
| 577 |
+
|
| 578 |
+
if delta_position_values is None:
|
| 579 |
+
# Fallback to the old hardcoded approach if metadata approach fails
|
| 580 |
+
if 'action.delta_eef_position' in data.columns:
|
| 581 |
+
delta_position_values = data['action.delta_eef_position'].to_numpy().tolist()
|
| 582 |
+
elif all(col in data.columns for col in ['action.x', 'action.y', 'action.z']):
|
| 583 |
+
x_vals = data['action.x'].to_numpy()
|
| 584 |
+
y_vals = data['action.y'].to_numpy()
|
| 585 |
+
z_vals = data['action.z'].to_numpy()
|
| 586 |
+
delta_position_values = np.column_stack((x_vals, y_vals, z_vals)).tolist()
|
| 587 |
+
else:
|
| 588 |
+
raise ValueError(f"No suitable position columns found. Available columns: {data.columns.tolist()}")
|
| 589 |
+
|
| 590 |
+
# Extract gripper data
|
| 591 |
+
gripper_values = None
|
| 592 |
+
gripper_candidates = ['gripper_close', 'gripper']
|
| 593 |
+
|
| 594 |
+
for grip_key in gripper_candidates:
|
| 595 |
+
full_key = f"action.{grip_key}"
|
| 596 |
+
if full_key in action_keys:
|
| 597 |
+
try:
|
| 598 |
+
le_action_cfg = self.lerobot_modality_meta.action
|
| 599 |
+
if grip_key in le_action_cfg:
|
| 600 |
+
le_key = le_action_cfg[grip_key].original_key or grip_key
|
| 601 |
+
if le_key in data.columns:
|
| 602 |
+
data_array = np.stack(data[le_key])
|
| 603 |
+
le_indices = np.arange(le_action_cfg[grip_key].start, le_action_cfg[grip_key].end)
|
| 604 |
+
gripper_data = data_array[:, le_indices].flatten()
|
| 605 |
+
gripper_values = gripper_data.tolist()
|
| 606 |
+
break
|
| 607 |
+
except Exception:
|
| 608 |
+
continue
|
| 609 |
+
|
| 610 |
+
if gripper_values is None:
|
| 611 |
+
# Fallback to the old hardcoded approach if metadata approach fails
|
| 612 |
+
if 'action.gripper_close' in data.columns:
|
| 613 |
+
gripper_values = data['action.gripper_close'].to_numpy().tolist()
|
| 614 |
+
elif 'action.gripper' in data.columns:
|
| 615 |
+
gripper_values = data['action.gripper'].to_numpy().tolist()
|
| 616 |
+
else:
|
| 617 |
+
raise ValueError(f"No suitable gripper columns found. Available columns: {data.columns.tolist()}")
|
| 618 |
+
|
| 619 |
+
return delta_position_values, gripper_values
|
| 620 |
+
|
| 621 |
+
def _get_modality_keys(self) -> dict:
|
| 622 |
+
"""Get the modality keys for the dataset.
|
| 623 |
+
The keys are the modality names, and the values are the keys for each modality.
|
| 624 |
+
See property `modality_keys` for the expected format.
|
| 625 |
+
"""
|
| 626 |
+
modality_keys = defaultdict(list)
|
| 627 |
+
for modality, config in self.modality_configs.items():
|
| 628 |
+
modality_keys[modality] = config.modality_keys
|
| 629 |
+
return modality_keys
|
| 630 |
+
|
| 631 |
+
def _get_delta_indices(self) -> dict[str, np.ndarray]:
|
| 632 |
+
"""Restructure the delta indices to use modality.key as keys instead of just the modalities."""
|
| 633 |
+
delta_indices: dict[str, np.ndarray] = {}
|
| 634 |
+
for config in self.modality_configs.values():
|
| 635 |
+
for key in config.modality_keys:
|
| 636 |
+
delta_indices[key] = np.array(config.delta_indices)
|
| 637 |
+
return delta_indices
|
| 638 |
+
|
| 639 |
+
def _get_lerobot_modality_meta(self) -> LeRobotModalityMetadata:
|
| 640 |
+
"""Get the metadata for the LeRobot dataset."""
|
| 641 |
+
modality_meta_path = self.dataset_path / LE_ROBOT_MODALITY_FILENAME
|
| 642 |
+
assert (
|
| 643 |
+
modality_meta_path.exists()
|
| 644 |
+
), f"Please provide a {LE_ROBOT_MODALITY_FILENAME} file in {self.dataset_path}"
|
| 645 |
+
with open(modality_meta_path, "r") as f:
|
| 646 |
+
modality_meta = LeRobotModalityMetadata.model_validate(json.load(f))
|
| 647 |
+
return modality_meta
|
| 648 |
+
|
| 649 |
+
def _get_lerobot_info_meta(self) -> dict:
|
| 650 |
+
"""Get the metadata for the LeRobot dataset."""
|
| 651 |
+
info_meta_path = self.dataset_path / LE_ROBOT_INFO_FILENAME
|
| 652 |
+
with open(info_meta_path, "r") as f:
|
| 653 |
+
info_meta = json.load(f)
|
| 654 |
+
return info_meta
|
| 655 |
+
|
| 656 |
+
def _get_data_path_pattern(self) -> str:
|
| 657 |
+
"""Get the data path pattern for the LeRobot dataset."""
|
| 658 |
+
return self.lerobot_info_meta["data_path"]
|
| 659 |
+
|
| 660 |
+
def _get_video_path_pattern(self) -> str:
|
| 661 |
+
"""Get the video path pattern for the LeRobot dataset."""
|
| 662 |
+
return self.lerobot_info_meta["video_path"]
|
| 663 |
+
|
| 664 |
+
def _get_chunk_size(self) -> int:
|
| 665 |
+
"""Get the chunk size for the LeRobot dataset."""
|
| 666 |
+
return self.lerobot_info_meta["chunks_size"]
|
| 667 |
+
|
| 668 |
+
def _get_tasks(self) -> pd.DataFrame:
|
| 669 |
+
"""Get the tasks for the dataset."""
|
| 670 |
+
tasks_path = self.dataset_path / LE_ROBOT_TASKS_FILENAME
|
| 671 |
+
with open(tasks_path, "r") as f:
|
| 672 |
+
tasks = [json.loads(line) for line in f]
|
| 673 |
+
df = pd.DataFrame(tasks)
|
| 674 |
+
return df.set_index("task_index")
|
| 675 |
+
|
| 676 |
+
def _check_integrity(self):
|
| 677 |
+
"""Use the config to check if the keys are valid and detect silent data corruption."""
|
| 678 |
+
ERROR_MSG_HEADER = f"Error occurred in initializing dataset {self.dataset_name}:\n"
|
| 679 |
+
|
| 680 |
+
for modality_config in self.modality_configs.values():
|
| 681 |
+
for key in modality_config.modality_keys:
|
| 682 |
+
if key == "lapa_action" or key == "dream_actions":
|
| 683 |
+
continue # no need for any metadata for lapa actions because it comes normalized
|
| 684 |
+
# Check if the key is valid
|
| 685 |
+
try:
|
| 686 |
+
self.lerobot_modality_meta.get_key_meta(key)
|
| 687 |
+
except Exception as e:
|
| 688 |
+
raise ValueError(
|
| 689 |
+
ERROR_MSG_HEADER + f"Unable to find key {key} in modality metadata:\n{e}"
|
| 690 |
+
)
|
| 691 |
+
|
| 692 |
+
def set_transforms_metadata(self, metadata: DatasetMetadata):
|
| 693 |
+
"""Set the metadata for the transforms. This is useful for transforms that need to know the metadata, such as the normalization values."""
|
| 694 |
+
self.transforms.set_metadata(metadata)
|
| 695 |
+
|
| 696 |
+
def set_epoch(self, epoch: int):
|
| 697 |
+
"""Set the epoch for the dataset.
|
| 698 |
+
|
| 699 |
+
Args:
|
| 700 |
+
epoch (int): The epoch to set.
|
| 701 |
+
"""
|
| 702 |
+
self.epoch = epoch
|
| 703 |
+
|
| 704 |
+
def __len__(self) -> int:
|
| 705 |
+
"""Get the total number of data points in the dataset.
|
| 706 |
+
|
| 707 |
+
Returns:
|
| 708 |
+
int: the total number of data points in the dataset.
|
| 709 |
+
"""
|
| 710 |
+
return len(self.all_steps)
|
| 711 |
+
|
| 712 |
+
def __str__(self) -> str:
|
| 713 |
+
"""Get the description of the dataset."""
|
| 714 |
+
return f"{self.dataset_name} ({len(self)} steps)"
|
| 715 |
+
|
| 716 |
+
|
| 717 |
+
def __getitem__(self, index: int) -> dict:
|
| 718 |
+
"""Get the data for a single step in a trajectory.
|
| 719 |
+
|
| 720 |
+
Args:
|
| 721 |
+
index (int): The index of the step to get.
|
| 722 |
+
|
| 723 |
+
Returns:
|
| 724 |
+
dict: The data for the step.
|
| 725 |
+
"""
|
| 726 |
+
trajectory_id, base_index = self.all_steps[index]
|
| 727 |
+
data = self.get_step_data(trajectory_id, base_index)
|
| 728 |
+
|
| 729 |
+
# Process all video keys dynamically
|
| 730 |
+
images = []
|
| 731 |
+
for video_key in self.modality_keys["video"]:
|
| 732 |
+
image = data[video_key][0]
|
| 733 |
+
|
| 734 |
+
# Apply image cropping if enabled and the video key is base_view
|
| 735 |
+
# Note: crop_obs_camera functionality has been removed
|
| 736 |
+
|
| 737 |
+
image = Image.fromarray(image).resize((224, 224))
|
| 738 |
+
images.append(image)
|
| 739 |
+
|
| 740 |
+
# Get language and action data
|
| 741 |
+
language = data[self.modality_keys["language"][0]][0]
|
| 742 |
+
action = []
|
| 743 |
+
for action_key in self.modality_keys["action"]:
|
| 744 |
+
action.append(data[action_key])
|
| 745 |
+
action = np.concatenate(action, axis=1)
|
| 746 |
+
|
| 747 |
+
return dict(action=action, image=images, language=language)
|
| 748 |
+
|
| 749 |
+
def get_step_data(self, trajectory_id: int, base_index: int) -> dict:
|
| 750 |
+
"""Get the RAW data for a single step in a trajectory. No transforms are applied.
|
| 751 |
+
|
| 752 |
+
Args:
|
| 753 |
+
trajectory_id (int): The name of the trajectory.
|
| 754 |
+
base_index (int): The base step index in the trajectory.
|
| 755 |
+
|
| 756 |
+
Returns:
|
| 757 |
+
dict: The RAW data for the step.
|
| 758 |
+
|
| 759 |
+
Example return:
|
| 760 |
+
{
|
| 761 |
+
"video": {
|
| 762 |
+
"video.image_side_0": [B, T, H, W, C],
|
| 763 |
+
"video.image_side_1": [B, T, H, W, C],
|
| 764 |
+
},
|
| 765 |
+
"state": {
|
| 766 |
+
"state.eef_position": [B, T, state_dim],
|
| 767 |
+
"state.eef_rotation": [B, T, state_dim],
|
| 768 |
+
},
|
| 769 |
+
"action": {
|
| 770 |
+
"action.eef_position": [B, T, action_dim],
|
| 771 |
+
"action.eef_rotation": [B, T, action_dim],
|
| 772 |
+
},
|
| 773 |
+
}
|
| 774 |
+
"""
|
| 775 |
+
data = {}
|
| 776 |
+
# Get the data for all modalities
|
| 777 |
+
self.curr_traj_data = self.get_trajectory_data(trajectory_id)
|
| 778 |
+
|
| 779 |
+
for modality in self.modality_keys:
|
| 780 |
+
# Get the data corresponding to each key in the modality
|
| 781 |
+
for key in self.modality_keys[modality]:
|
| 782 |
+
data[key] = self.get_data_by_modality(trajectory_id, modality, key, base_index)
|
| 783 |
+
return data
|
| 784 |
+
|
| 785 |
+
def _preload_all_trajectories(self):
|
| 786 |
+
"""预加载所有trajectories到内存"""
|
| 787 |
+
print("Preloading all trajectories...")
|
| 788 |
+
self._trajectory_cache = {}
|
| 789 |
+
for trajectory_id in tqdm(self.trajectory_ids, desc="Loading trajectories"):
|
| 790 |
+
chunk_index = self.get_episode_chunk(trajectory_id)
|
| 791 |
+
parquet_path = self.dataset_path / self.data_path_pattern.format(
|
| 792 |
+
episode_chunk=chunk_index, episode_index=trajectory_id
|
| 793 |
+
)
|
| 794 |
+
self._trajectory_cache[trajectory_id] = pd.read_parquet(parquet_path)
|
| 795 |
+
print(f"Preloaded {len(self._trajectory_cache)} trajectories")
|
| 796 |
+
|
| 797 |
+
def get_trajectory_data(self, trajectory_id: int) -> pd.DataFrame:
|
| 798 |
+
"""Get the data for a trajectory."""
|
| 799 |
+
|
| 800 |
+
if self.preload_all:
|
| 801 |
+
return self._trajectory_cache[trajectory_id]
|
| 802 |
+
else:
|
| 803 |
+
if self.curr_traj_id == trajectory_id and self.curr_traj_data is not None:
|
| 804 |
+
return self.curr_traj_data
|
| 805 |
+
else:
|
| 806 |
+
chunk_index = self.get_episode_chunk(trajectory_id)
|
| 807 |
+
parquet_path = self.dataset_path / self.data_path_pattern.format(
|
| 808 |
+
episode_chunk=chunk_index, episode_index=trajectory_id
|
| 809 |
+
)
|
| 810 |
+
assert parquet_path.exists(), f"Parquet file not found at {parquet_path}"
|
| 811 |
+
return pd.read_parquet(parquet_path)
|
| 812 |
+
|
| 813 |
+
def get_trajectory_index(self, trajectory_id: int) -> int:
|
| 814 |
+
"""Get the index of the trajectory in the dataset by the trajectory ID.
|
| 815 |
+
This is useful when you need to get the trajectory length or sampling weight corresponding to the trajectory ID.
|
| 816 |
+
|
| 817 |
+
Args:
|
| 818 |
+
trajectory_id (str): The ID of the trajectory.
|
| 819 |
+
|
| 820 |
+
Returns:
|
| 821 |
+
int: The index of the trajectory in the dataset.
|
| 822 |
+
"""
|
| 823 |
+
trajectory_indices = np.where(self.trajectory_ids == trajectory_id)[0]
|
| 824 |
+
if len(trajectory_indices) != 1:
|
| 825 |
+
raise ValueError(
|
| 826 |
+
f"Error finding trajectory index for {trajectory_id}, found {trajectory_indices}"
|
| 827 |
+
)
|
| 828 |
+
return trajectory_indices[0]
|
| 829 |
+
|
| 830 |
+
def get_episode_chunk(self, ep_index: int) -> int:
|
| 831 |
+
"""Get the chunk index for an episode index."""
|
| 832 |
+
return ep_index // self.chunk_size
|
| 833 |
+
|
| 834 |
+
def retrieve_data_and_pad(
|
| 835 |
+
self,
|
| 836 |
+
array: np.ndarray,
|
| 837 |
+
step_indices: np.ndarray,
|
| 838 |
+
max_length: int,
|
| 839 |
+
padding_strategy: str = "first_last",
|
| 840 |
+
) -> np.ndarray:
|
| 841 |
+
"""Retrieve the data from the dataset and pad it if necessary.
|
| 842 |
+
Args:
|
| 843 |
+
array (np.ndarray): The array to retrieve the data from.
|
| 844 |
+
step_indices (np.ndarray): The step indices to retrieve the data for.
|
| 845 |
+
max_length (int): The maximum length of the data.
|
| 846 |
+
padding_strategy (str): The padding strategy, either "first" or "last".
|
| 847 |
+
"""
|
| 848 |
+
# Get the padding indices
|
| 849 |
+
front_padding_indices = step_indices < 0
|
| 850 |
+
end_padding_indices = step_indices >= max_length
|
| 851 |
+
padding_positions = np.logical_or(front_padding_indices, end_padding_indices)
|
| 852 |
+
# Retrieve the data with the non-padding indices
|
| 853 |
+
# If there exists some padding, Given T step_indices, the shape of the retrieved data will be (T', ...) where T' < T
|
| 854 |
+
raw_data = array[step_indices[~padding_positions]]
|
| 855 |
+
assert isinstance(raw_data, np.ndarray), f"{type(raw_data)=}"
|
| 856 |
+
# This is the shape of the output, (T, ...)
|
| 857 |
+
if raw_data.ndim == 1:
|
| 858 |
+
expected_shape = (len(step_indices),)
|
| 859 |
+
else:
|
| 860 |
+
expected_shape = (len(step_indices), *array.shape[1:])
|
| 861 |
+
|
| 862 |
+
# Pad the data
|
| 863 |
+
output = np.zeros(expected_shape)
|
| 864 |
+
# Assign the non-padded data
|
| 865 |
+
output[~padding_positions] = raw_data
|
| 866 |
+
# If there exists some padding, pad the data
|
| 867 |
+
if padding_positions.any():
|
| 868 |
+
if padding_strategy == "first_last":
|
| 869 |
+
# Use first / last step data to pad
|
| 870 |
+
front_padding_data = array[0]
|
| 871 |
+
end_padding_data = array[-1]
|
| 872 |
+
output[front_padding_indices] = front_padding_data
|
| 873 |
+
output[end_padding_indices] = end_padding_data
|
| 874 |
+
elif padding_strategy == "zero":
|
| 875 |
+
# Use zero padding
|
| 876 |
+
output[padding_positions] = 0
|
| 877 |
+
else:
|
| 878 |
+
raise ValueError(f"Invalid padding strategy: {padding_strategy}")
|
| 879 |
+
return output
|
| 880 |
+
|
| 881 |
+
def get_video_path(self, trajectory_id: int, key: str) -> Path:
|
| 882 |
+
chunk_index = self.get_episode_chunk(trajectory_id)
|
| 883 |
+
original_key = self.lerobot_modality_meta.video[key].original_key
|
| 884 |
+
if original_key is None:
|
| 885 |
+
original_key = key
|
| 886 |
+
video_filename = self.video_path_pattern.format(
|
| 887 |
+
episode_chunk=chunk_index, episode_index=trajectory_id, video_key=original_key
|
| 888 |
+
)
|
| 889 |
+
return self.dataset_path / video_filename
|
| 890 |
+
|
| 891 |
+
def get_video(
|
| 892 |
+
self,
|
| 893 |
+
trajectory_id: int,
|
| 894 |
+
key: str,
|
| 895 |
+
base_index: int,
|
| 896 |
+
) -> np.ndarray:
|
| 897 |
+
"""Get the video frames for a trajectory by a base index.
|
| 898 |
+
|
| 899 |
+
Args:
|
| 900 |
+
dataset (BaseSingleDataset): The dataset to retrieve the data from.
|
| 901 |
+
trajectory_id (str): The ID of the trajectory.
|
| 902 |
+
key (str): The key of the video.
|
| 903 |
+
base_index (int): The base index of the trajectory.
|
| 904 |
+
|
| 905 |
+
Returns:
|
| 906 |
+
np.ndarray: The video frames for the trajectory and frame indices. Shape: (T, H, W, C)
|
| 907 |
+
"""
|
| 908 |
+
# Get the step indices
|
| 909 |
+
step_indices = self.delta_indices[key] + base_index
|
| 910 |
+
# print(f"{step_indices=}")
|
| 911 |
+
# Get the trajectory index
|
| 912 |
+
trajectory_index = self.get_trajectory_index(trajectory_id)
|
| 913 |
+
# Ensure the indices are within the valid range
|
| 914 |
+
# This is equivalent to padding the video with extra frames at the beginning and end
|
| 915 |
+
step_indices = np.maximum(step_indices, 0)
|
| 916 |
+
step_indices = np.minimum(step_indices, self.trajectory_lengths[trajectory_index] - 1)
|
| 917 |
+
assert key.startswith("video."), f"Video key must start with 'video.', got {key}"
|
| 918 |
+
# Get the sub-key
|
| 919 |
+
key = key.replace("video.", "")
|
| 920 |
+
video_path = self.get_video_path(trajectory_id, key)
|
| 921 |
+
# Get the action/state timestamps for each frame in the video
|
| 922 |
+
assert self.curr_traj_data is not None, f"No data found for {trajectory_id}"
|
| 923 |
+
assert "timestamp" in self.curr_traj_data.columns, f"No timestamp found in {trajectory_id}"
|
| 924 |
+
timestamp: np.ndarray = self.curr_traj_data["timestamp"].to_numpy()
|
| 925 |
+
# Get the corresponding video timestamps from the step indices
|
| 926 |
+
video_timestamp = timestamp[step_indices]
|
| 927 |
+
|
| 928 |
+
return get_frames_by_timestamps(
|
| 929 |
+
video_path.as_posix(),
|
| 930 |
+
video_timestamp,
|
| 931 |
+
video_backend=self.video_backend,
|
| 932 |
+
video_backend_kwargs=self.video_backend_kwargs,
|
| 933 |
+
step_indices_for_assert=step_indices,
|
| 934 |
+
)
|
| 935 |
+
|
| 936 |
+
def get_state_or_action(
|
| 937 |
+
self,
|
| 938 |
+
trajectory_id: int,
|
| 939 |
+
modality: str,
|
| 940 |
+
key: str,
|
| 941 |
+
base_index: int,
|
| 942 |
+
) -> np.ndarray:
|
| 943 |
+
"""Get the state or action data for a trajectory by a base index.
|
| 944 |
+
If the step indices are out of range, pad with the data:
|
| 945 |
+
if the data is stored in absolute format, pad with the first or last step data;
|
| 946 |
+
otherwise, pad with zero.
|
| 947 |
+
|
| 948 |
+
Args:
|
| 949 |
+
dataset (BaseSingleDataset): The dataset to retrieve the data from.
|
| 950 |
+
trajectory_id (int): The ID of the trajectory.
|
| 951 |
+
modality (str): The modality of the data.
|
| 952 |
+
key (str): The key of the data.
|
| 953 |
+
base_index (int): The base index of the trajectory.
|
| 954 |
+
|
| 955 |
+
Returns:
|
| 956 |
+
np.ndarray: The data for the trajectory and step indices.
|
| 957 |
+
"""
|
| 958 |
+
# Get the step indices
|
| 959 |
+
step_indices = self.delta_indices[key] + base_index
|
| 960 |
+
# Get the trajectory index
|
| 961 |
+
trajectory_index = self.get_trajectory_index(trajectory_id)
|
| 962 |
+
# Get the maximum length of the trajectory
|
| 963 |
+
max_length = self.trajectory_lengths[trajectory_index]
|
| 964 |
+
assert key.startswith(modality + "."), f"{key} must start with {modality + '.'}, got {key}"
|
| 965 |
+
# Get the sub-key, e.g. state.joint_angles -> joint_angles
|
| 966 |
+
key = key.replace(modality + ".", "")
|
| 967 |
+
# Get the lerobot key
|
| 968 |
+
le_state_or_action_cfg = getattr(self.lerobot_modality_meta, modality)
|
| 969 |
+
le_key = le_state_or_action_cfg[key].original_key
|
| 970 |
+
if le_key is None:
|
| 971 |
+
le_key = key
|
| 972 |
+
# Get the data array, shape: (T, D)
|
| 973 |
+
assert self.curr_traj_data is not None, f"No data found for {trajectory_id}"
|
| 974 |
+
assert le_key in self.curr_traj_data.columns, f"No {le_key} found in {trajectory_id}"
|
| 975 |
+
data_array: np.ndarray = np.stack(self.curr_traj_data[le_key]) # type: ignore
|
| 976 |
+
assert data_array.ndim == 2, f"Expected 2D array, got key {le_key} is{data_array.shape} array"
|
| 977 |
+
le_indices = np.arange(
|
| 978 |
+
le_state_or_action_cfg[key].start,
|
| 979 |
+
le_state_or_action_cfg[key].end,
|
| 980 |
+
)
|
| 981 |
+
data_array = data_array[:, le_indices]
|
| 982 |
+
# Get the state or action configuration
|
| 983 |
+
state_or_action_cfg = getattr(self.metadata.modalities, modality)[key]
|
| 984 |
+
|
| 985 |
+
# Pad the data
|
| 986 |
+
return self.retrieve_data_and_pad(
|
| 987 |
+
array=data_array,
|
| 988 |
+
step_indices=step_indices,
|
| 989 |
+
max_length=max_length,
|
| 990 |
+
padding_strategy="first_last" if state_or_action_cfg.absolute else "zero",
|
| 991 |
+
# padding_strategy="first_last", # HACK for realdata
|
| 992 |
+
)
|
| 993 |
+
|
| 994 |
+
def get_language(
|
| 995 |
+
self,
|
| 996 |
+
trajectory_id: int,
|
| 997 |
+
key: str,
|
| 998 |
+
base_index: int,
|
| 999 |
+
) -> list[str]:
|
| 1000 |
+
"""Get the language annotation data for a trajectory by step indices.
|
| 1001 |
+
|
| 1002 |
+
Args:
|
| 1003 |
+
dataset (BaseSingleDataset): The dataset to retrieve the data from.
|
| 1004 |
+
trajectory_id (int): The ID of the trajectory.
|
| 1005 |
+
key (str): The key of the annotation.
|
| 1006 |
+
base_index (int): The base index of the trajectory.
|
| 1007 |
+
|
| 1008 |
+
Returns:
|
| 1009 |
+
list[str]: The annotation data for the trajectory and step indices. If no matching data is found, return empty strings.
|
| 1010 |
+
"""
|
| 1011 |
+
assert self.curr_traj_data is not None, f"No data found for {trajectory_id}"
|
| 1012 |
+
# Get the step indices
|
| 1013 |
+
step_indices = self.delta_indices[key] + base_index
|
| 1014 |
+
# Get the trajectory index
|
| 1015 |
+
trajectory_index = self.get_trajectory_index(trajectory_id)
|
| 1016 |
+
# Get the maximum length of the trajectory
|
| 1017 |
+
max_length = self.trajectory_lengths[trajectory_index]
|
| 1018 |
+
# Get the end times corresponding to the closest indices
|
| 1019 |
+
step_indices = np.maximum(step_indices, 0)
|
| 1020 |
+
step_indices = np.minimum(step_indices, max_length - 1)
|
| 1021 |
+
# Get the annotations
|
| 1022 |
+
task_indices: list[int] = []
|
| 1023 |
+
assert key.startswith(
|
| 1024 |
+
"annotation."
|
| 1025 |
+
), f"Language key must start with 'annotation.', got {key}"
|
| 1026 |
+
subkey = key.replace("annotation.", "")
|
| 1027 |
+
annotation_meta = self.lerobot_modality_meta.annotation
|
| 1028 |
+
assert annotation_meta is not None, f"Annotation metadata is None for {subkey}"
|
| 1029 |
+
assert (
|
| 1030 |
+
subkey in annotation_meta
|
| 1031 |
+
), f"Annotation key {subkey} not found in metadata, available annotation keys: {annotation_meta.keys()}"
|
| 1032 |
+
subkey_meta = annotation_meta[subkey]
|
| 1033 |
+
original_key = subkey_meta.original_key
|
| 1034 |
+
if original_key is None:
|
| 1035 |
+
original_key = key
|
| 1036 |
+
for i in range(len(step_indices)):
|
| 1037 |
+
task_indices.append(self.curr_traj_data[original_key][step_indices[i]].item())
|
| 1038 |
+
return self.tasks.loc[task_indices]["description"].tolist() # 适配zhenyu脚本
|
| 1039 |
+
|
| 1040 |
+
def get_data_by_modality(
|
| 1041 |
+
self,
|
| 1042 |
+
trajectory_id: int,
|
| 1043 |
+
modality: str,
|
| 1044 |
+
key: str,
|
| 1045 |
+
base_index: int,
|
| 1046 |
+
):
|
| 1047 |
+
"""Get the data corresponding to the modality for a trajectory by a base index.
|
| 1048 |
+
This method will call the corresponding helper method based on the modality.
|
| 1049 |
+
See the helper methods for more details.
|
| 1050 |
+
NOTE: For the language modality, the data is padded with empty strings if no matching data is found.
|
| 1051 |
+
|
| 1052 |
+
Args:
|
| 1053 |
+
dataset (BaseSingleDataset): The dataset to retrieve the data from.
|
| 1054 |
+
trajectory_id (int): The ID of the trajectory.
|
| 1055 |
+
modality (str): The modality of the data.
|
| 1056 |
+
key (str): The key of the data.
|
| 1057 |
+
base_index (int): The base index of the trajectory.
|
| 1058 |
+
"""
|
| 1059 |
+
if modality == "video":
|
| 1060 |
+
return self.get_video(trajectory_id, key, base_index)
|
| 1061 |
+
elif modality == "state" or modality == "action":
|
| 1062 |
+
return self.get_state_or_action(trajectory_id, modality, key, base_index)
|
| 1063 |
+
elif modality == "language":
|
| 1064 |
+
return self.get_language(trajectory_id, key, base_index)
|
| 1065 |
+
else:
|
| 1066 |
+
raise ValueError(f"Invalid modality: {modality}")
|
| 1067 |
+
|
| 1068 |
+
def save_dataset_statistics(self, save_path: Path | str, format: str = "json") -> None:
|
| 1069 |
+
"""
|
| 1070 |
+
Save dataset statistics to specified path in the required format.
|
| 1071 |
+
Only includes statistics for keys that are actually used in the dataset.
|
| 1072 |
+
Gripper-related keys will be placed at the end.
|
| 1073 |
+
|
| 1074 |
+
Args:
|
| 1075 |
+
save_path (Path | str): Path to save the statistics file
|
| 1076 |
+
format (str): Save format, currently only supports "json"
|
| 1077 |
+
"""
|
| 1078 |
+
save_path = Path(save_path)
|
| 1079 |
+
save_path.parent.mkdir(parents=True, exist_ok=True)
|
| 1080 |
+
|
| 1081 |
+
# Build the data structure to save
|
| 1082 |
+
statistics_data = {}
|
| 1083 |
+
|
| 1084 |
+
# Get used modality keys
|
| 1085 |
+
used_action_keys, used_state_keys = get_used_modality_keys(self.modality_keys)
|
| 1086 |
+
|
| 1087 |
+
# Organize statistics by tag
|
| 1088 |
+
tag = self.tag
|
| 1089 |
+
tag_stats = {}
|
| 1090 |
+
|
| 1091 |
+
# Process action statistics (only for used keys)
|
| 1092 |
+
if hasattr(self.metadata.statistics, 'action') and self.metadata.statistics.action:
|
| 1093 |
+
action_stats = self.metadata.statistics.action
|
| 1094 |
+
|
| 1095 |
+
# Filter to only include used action keys and reorder: non-gripper first, gripper last
|
| 1096 |
+
non_gripper_keys = []
|
| 1097 |
+
gripper_keys = []
|
| 1098 |
+
|
| 1099 |
+
for key in action_stats.keys():
|
| 1100 |
+
if key in used_action_keys:
|
| 1101 |
+
if "gripper" in key.lower():
|
| 1102 |
+
gripper_keys.append(key)
|
| 1103 |
+
else:
|
| 1104 |
+
non_gripper_keys.append(key)
|
| 1105 |
+
|
| 1106 |
+
# Reorder: non-gripper first, gripper last
|
| 1107 |
+
reordered_keys = non_gripper_keys + gripper_keys
|
| 1108 |
+
|
| 1109 |
+
filtered_action_stats = {}
|
| 1110 |
+
for key in reordered_keys:
|
| 1111 |
+
filtered_action_stats[key] = action_stats[key]
|
| 1112 |
+
|
| 1113 |
+
if filtered_action_stats:
|
| 1114 |
+
# Combine statistics from filtered action sub-keys
|
| 1115 |
+
combined_action_stats = combine_modality_stats(filtered_action_stats)
|
| 1116 |
+
|
| 1117 |
+
# Add mask field: based on whether it's gripper or not
|
| 1118 |
+
mask = generate_action_mask_for_used_keys(
|
| 1119 |
+
self.metadata.modalities.action, filtered_action_stats.keys()
|
| 1120 |
+
)
|
| 1121 |
+
combined_action_stats["mask"] = mask
|
| 1122 |
+
|
| 1123 |
+
tag_stats["action"] = combined_action_stats
|
| 1124 |
+
|
| 1125 |
+
# Process state statistics (only for used keys)
|
| 1126 |
+
if hasattr(self.metadata.statistics, 'state') and self.metadata.statistics.state:
|
| 1127 |
+
state_stats = self.metadata.statistics.state
|
| 1128 |
+
|
| 1129 |
+
# Filter to only include used state keys, optionally reorder gripper to end
|
| 1130 |
+
non_gripper_keys = []
|
| 1131 |
+
gripper_keys = []
|
| 1132 |
+
|
| 1133 |
+
for key in state_stats.keys():
|
| 1134 |
+
if key in used_state_keys:
|
| 1135 |
+
if "gripper" in key.lower():
|
| 1136 |
+
gripper_keys.append(key)
|
| 1137 |
+
else:
|
| 1138 |
+
non_gripper_keys.append(key)
|
| 1139 |
+
|
| 1140 |
+
# Reorder: non-gripper first, gripper last
|
| 1141 |
+
reordered_keys = non_gripper_keys + gripper_keys
|
| 1142 |
+
|
| 1143 |
+
filtered_state_stats = {}
|
| 1144 |
+
for key in reordered_keys:
|
| 1145 |
+
filtered_state_stats[key] = state_stats[key]
|
| 1146 |
+
|
| 1147 |
+
if filtered_state_stats:
|
| 1148 |
+
combined_state_stats = combine_modality_stats(filtered_state_stats)
|
| 1149 |
+
tag_stats["state"] = combined_state_stats
|
| 1150 |
+
|
| 1151 |
+
# Add dataset counts
|
| 1152 |
+
tag_stats["num_transitions"] = len(self)
|
| 1153 |
+
tag_stats["num_trajectories"] = len(self.trajectory_ids)
|
| 1154 |
+
|
| 1155 |
+
statistics_data[tag] = tag_stats
|
| 1156 |
+
|
| 1157 |
+
# Save as JSON file
|
| 1158 |
+
if format.lower() == "json":
|
| 1159 |
+
if not str(save_path).endswith('.json'):
|
| 1160 |
+
save_path = save_path.with_suffix('.json')
|
| 1161 |
+
with open(save_path, 'w', encoding='utf-8') as f:
|
| 1162 |
+
json.dump(statistics_data, f, indent=2, ensure_ascii=False)
|
| 1163 |
+
else:
|
| 1164 |
+
raise ValueError(f"Unsupported format: {format}. Currently only 'json' is supported.")
|
| 1165 |
+
|
| 1166 |
+
print(f"Single dataset statistics saved to: {save_path}")
|
| 1167 |
+
print(f"Used action keys (reordered): {list(used_action_keys)}")
|
| 1168 |
+
print(f"Used state keys (reordered): {list(used_state_keys)}")
|
| 1169 |
+
|
| 1170 |
+
|
| 1171 |
+
class CachedLeRobotSingleDataset(LeRobotSingleDataset):
|
| 1172 |
+
def __init__(self, img_resize: tuple[int, int] | None = None, *args, **kwargs):
|
| 1173 |
+
"""
|
| 1174 |
+
This class caches the video frames for each trajectory and key.
|
| 1175 |
+
It is recommended to use this class if the video frames need to be accessed multiple times.
|
| 1176 |
+
|
| 1177 |
+
Args:
|
| 1178 |
+
resize_img (tuple[int, int], optional): The size to resize the video frames to reduce memory usage.
|
| 1179 |
+
"""
|
| 1180 |
+
# Convert img_resize to tuple if it is not already
|
| 1181 |
+
if img_resize is not None and not isinstance(img_resize, tuple):
|
| 1182 |
+
img_resize = tuple(img_resize)
|
| 1183 |
+
assert len(img_resize) == 2, f"Expected tuple of length 2, got {img_resize}"
|
| 1184 |
+
self.img_resize = img_resize
|
| 1185 |
+
|
| 1186 |
+
# Initialize img_resize attribute first to ensure it exists
|
| 1187 |
+
super().__init__(*args, **kwargs)
|
| 1188 |
+
cached_frames: dict[str, np.ndarray] = {}
|
| 1189 |
+
|
| 1190 |
+
for key in self.modality_keys["video"]:
|
| 1191 |
+
all_frames = []
|
| 1192 |
+
original_key = key
|
| 1193 |
+
key = key.replace("video.", "")
|
| 1194 |
+
for trajectory_id, trajectory_length in tqdm(
|
| 1195 |
+
zip(self.trajectory_ids, self.trajectory_lengths),
|
| 1196 |
+
total=len(self.trajectory_ids),
|
| 1197 |
+
desc=f"Caching {key} frames",
|
| 1198 |
+
):
|
| 1199 |
+
video_path = self.get_video_path(trajectory_id, key)
|
| 1200 |
+
frames = get_all_frames(
|
| 1201 |
+
video_path.as_posix(),
|
| 1202 |
+
video_backend=self.video_backend,
|
| 1203 |
+
video_backend_kwargs=self.video_backend_kwargs,
|
| 1204 |
+
resize_size=img_resize,
|
| 1205 |
+
)
|
| 1206 |
+
assert frames.ndim == 4, f"Expected 4D array, got {frames.shape} array"
|
| 1207 |
+
assert frames.shape[3] == 3, f"Expected 3 channels, got {frames.shape[3]} channels"
|
| 1208 |
+
|
| 1209 |
+
# Apply image cropping if enabled and the video key is base_view
|
| 1210 |
+
# Note: crop_obs_camera functionality has been removed
|
| 1211 |
+
|
| 1212 |
+
# assert (
|
| 1213 |
+
# frames.shape[0] == trajectory_length
|
| 1214 |
+
# ), f"Expected {trajectory_length} frames, got {frames.shape[0]} frames"
|
| 1215 |
+
all_frames.append(frames)
|
| 1216 |
+
cached_frames[key] = np.concatenate(all_frames, axis=0)
|
| 1217 |
+
print(f"{key}: {cached_frames[key].shape}")
|
| 1218 |
+
self.cached_frames = cached_frames
|
| 1219 |
+
self.start_indices = np.cumsum(self.trajectory_lengths) - self.trajectory_lengths
|
| 1220 |
+
|
| 1221 |
+
def get_video(self, trajectory_id: int, key: str, base_index: int) -> np.ndarray:
|
| 1222 |
+
step_indices = self.delta_indices[key] + base_index
|
| 1223 |
+
# Get the trajectory index
|
| 1224 |
+
trajectory_index = self.get_trajectory_index(trajectory_id)
|
| 1225 |
+
# Ensure the indices are within the valid range
|
| 1226 |
+
# This is equivalent to padding the video with extra frames at the beginning and end
|
| 1227 |
+
step_indices = np.maximum(step_indices, 0)
|
| 1228 |
+
step_indices = np.minimum(step_indices, self.trajectory_lengths[trajectory_index] - 1)
|
| 1229 |
+
assert key.startswith("video."), f"Video key must start with 'video.', got {key}"
|
| 1230 |
+
# Get the sub-key
|
| 1231 |
+
key = key.replace("video.", "")
|
| 1232 |
+
# Calculate the absolute indices
|
| 1233 |
+
absolute_indices = self.start_indices[trajectory_index] + step_indices
|
| 1234 |
+
return self.cached_frames[key][absolute_indices]
|
| 1235 |
+
|
| 1236 |
+
def get_step_data(self, trajectory_id: int, base_index: int) -> dict:
|
| 1237 |
+
"""Get the RAW data for a single step. No transforms are applied.
|
| 1238 |
+
|
| 1239 |
+
Args:
|
| 1240 |
+
trajectory_id (str): The ID of the trajectory.
|
| 1241 |
+
base_index (int): The base index of the step.
|
| 1242 |
+
|
| 1243 |
+
Returns:
|
| 1244 |
+
dict: The data for the step.
|
| 1245 |
+
"""
|
| 1246 |
+
data = {}
|
| 1247 |
+
self.curr_traj_data = self.get_trajectory_data(trajectory_id)
|
| 1248 |
+
# Get the data for all modalities
|
| 1249 |
+
for modality in self.modality_keys:
|
| 1250 |
+
# Get the data corresponding to each key in the modality
|
| 1251 |
+
for key in self.modality_keys[modality]:
|
| 1252 |
+
data[key] = self.get_data_by_modality(trajectory_id, modality, key, base_index)
|
| 1253 |
+
return data
|
| 1254 |
+
|
| 1255 |
+
def set_transforms_metadata(self, metadata: DatasetMetadata):
|
| 1256 |
+
"""Set the metadata for the transforms. This is useful for transforms that need to know the metadata, such as the normalization values."""
|
| 1257 |
+
if self.img_resize is not None:
|
| 1258 |
+
all_video_keys = [key for key in self.modality_keys["video"]]
|
| 1259 |
+
for key in metadata.modalities.video:
|
| 1260 |
+
if key in all_video_keys:
|
| 1261 |
+
metadata.modalities.video[key].resolution = self.img_resize
|
| 1262 |
+
super().set_transforms_metadata(metadata)
|
| 1263 |
+
|
| 1264 |
+
|
| 1265 |
+
def safe_hash(input_tuple):
|
| 1266 |
+
# keep 128 bits of the hash
|
| 1267 |
+
tuple_string = repr(input_tuple).encode("utf-8")
|
| 1268 |
+
sha256 = hashlib.sha256()
|
| 1269 |
+
sha256.update(tuple_string)
|
| 1270 |
+
|
| 1271 |
+
seed = int(sha256.hexdigest(), 16)
|
| 1272 |
+
|
| 1273 |
+
return seed & 0xFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
|
| 1274 |
+
|
| 1275 |
+
|
| 1276 |
+
class MixtureSpecElement(BaseModel):
|
| 1277 |
+
dataset_path: list[Path] | Path = Field(..., description="The path to the dataset.")
|
| 1278 |
+
dataset_weight: float = Field(..., description="The weight of the dataset in the mixture.")
|
| 1279 |
+
distribute_weights: bool = Field(
|
| 1280 |
+
default=False,
|
| 1281 |
+
description="Whether to distribute the weights of the dataset across all the paths. If True, the weights will be evenly distributed across all the paths.",
|
| 1282 |
+
)
|
| 1283 |
+
|
| 1284 |
+
|
| 1285 |
+
# Helper functions for dataset statistics
|
| 1286 |
+
|
| 1287 |
+
def combine_modality_stats(modality_stats: dict) -> dict:
|
| 1288 |
+
"""
|
| 1289 |
+
Combine statistics from all sub-keys under a modality.
|
| 1290 |
+
|
| 1291 |
+
Args:
|
| 1292 |
+
modality_stats (dict): Statistics for a modality, containing multiple sub-keys.
|
| 1293 |
+
Each sub-key contains DatasetStatisticalValues object.
|
| 1294 |
+
|
| 1295 |
+
Returns:
|
| 1296 |
+
dict: Combined statistics
|
| 1297 |
+
"""
|
| 1298 |
+
combined_stats = {
|
| 1299 |
+
"mean": [],
|
| 1300 |
+
"std": [],
|
| 1301 |
+
"max": [],
|
| 1302 |
+
"min": [],
|
| 1303 |
+
"q01": [],
|
| 1304 |
+
"q99": []
|
| 1305 |
+
}
|
| 1306 |
+
|
| 1307 |
+
# Combine statistics in sub-key order
|
| 1308 |
+
for subkey in modality_stats.keys():
|
| 1309 |
+
subkey_stats = modality_stats[subkey] # This is a DatasetStatisticalValues object
|
| 1310 |
+
|
| 1311 |
+
# Convert DatasetStatisticalValues to dict-like access
|
| 1312 |
+
for stat_name in ["mean", "std", "max", "min", "q01", "q99"]:
|
| 1313 |
+
stat_value = getattr(subkey_stats, stat_name)
|
| 1314 |
+
if isinstance(stat_value, (list, tuple)):
|
| 1315 |
+
combined_stats[stat_name].extend(stat_value)
|
| 1316 |
+
else:
|
| 1317 |
+
# Handle NDArray case - convert to list
|
| 1318 |
+
if hasattr(stat_value, 'tolist'):
|
| 1319 |
+
combined_stats[stat_name].extend(stat_value.tolist())
|
| 1320 |
+
else:
|
| 1321 |
+
combined_stats[stat_name].append(float(stat_value))
|
| 1322 |
+
|
| 1323 |
+
return combined_stats
|
| 1324 |
+
|
| 1325 |
+
def generate_action_mask_for_used_keys(action_modalities: dict, used_action_keys_ordered) -> list[bool]:
|
| 1326 |
+
"""
|
| 1327 |
+
Generate mask based on action modalities, but only for used keys.
|
| 1328 |
+
Gripper-related are False, others are True.
|
| 1329 |
+
|
| 1330 |
+
Args:
|
| 1331 |
+
action_modalities (dict): Configuration information for action modalities.
|
| 1332 |
+
used_action_keys_ordered: Iterable of actually used action keys in the correct order.
|
| 1333 |
+
|
| 1334 |
+
Returns:
|
| 1335 |
+
list[bool]: List of mask values
|
| 1336 |
+
"""
|
| 1337 |
+
mask = []
|
| 1338 |
+
|
| 1339 |
+
# Generate mask in the same order as the statistics were combined
|
| 1340 |
+
for subkey in used_action_keys_ordered:
|
| 1341 |
+
if subkey in action_modalities:
|
| 1342 |
+
subkey_config = action_modalities[subkey]
|
| 1343 |
+
|
| 1344 |
+
# Get dimension count from shape
|
| 1345 |
+
if hasattr(subkey_config, 'shape') and len(subkey_config.shape) > 0:
|
| 1346 |
+
dim_count = subkey_config.shape[0]
|
| 1347 |
+
else:
|
| 1348 |
+
dim_count = 1
|
| 1349 |
+
|
| 1350 |
+
# Check if it's gripper-related
|
| 1351 |
+
is_gripper = "gripper" in subkey.lower()
|
| 1352 |
+
|
| 1353 |
+
# Generate mask value for each dimension
|
| 1354 |
+
for _ in range(dim_count):
|
| 1355 |
+
mask.append(not is_gripper) # gripper is False, others are True
|
| 1356 |
+
|
| 1357 |
+
return mask
|
| 1358 |
+
|
| 1359 |
+
def get_used_modality_keys(modality_keys: dict) -> tuple[set, set]:
|
| 1360 |
+
"""Extract used action and state keys from modality configuration."""
|
| 1361 |
+
used_action_keys = set()
|
| 1362 |
+
used_state_keys = set()
|
| 1363 |
+
|
| 1364 |
+
# Extract action keys (remove "action." prefix)
|
| 1365 |
+
for action_key in modality_keys.get("action", []):
|
| 1366 |
+
if action_key.startswith("action."):
|
| 1367 |
+
clean_key = action_key.replace("action.", "")
|
| 1368 |
+
used_action_keys.add(clean_key)
|
| 1369 |
+
|
| 1370 |
+
# Extract state keys (remove "state." prefix)
|
| 1371 |
+
for state_key in modality_keys.get("state", []):
|
| 1372 |
+
if state_key.startswith("state."):
|
| 1373 |
+
clean_key = state_key.replace("state.", "")
|
| 1374 |
+
used_state_keys.add(clean_key)
|
| 1375 |
+
|
| 1376 |
+
return used_action_keys, used_state_keys
|
| 1377 |
+
|
| 1378 |
+
class LeRobotMixtureDataset(Dataset):
|
| 1379 |
+
"""
|
| 1380 |
+
A mixture of multiple datasets. This class samples a single dataset based on the dataset weights and then calls the `__getitem__` method of the sampled dataset.
|
| 1381 |
+
It is recommended to modify the single dataset class instead of this class.
|
| 1382 |
+
"""
|
| 1383 |
+
|
| 1384 |
+
def __init__(
|
| 1385 |
+
self,
|
| 1386 |
+
data_mixture: Sequence[tuple[LeRobotSingleDataset, float]],
|
| 1387 |
+
mode: str,
|
| 1388 |
+
balance_dataset_weights: bool = True,
|
| 1389 |
+
balance_trajectory_weights: bool = True,
|
| 1390 |
+
seed: int = 42,
|
| 1391 |
+
metadata_config: dict = {
|
| 1392 |
+
"percentile_mixing_method": "min_max",
|
| 1393 |
+
},
|
| 1394 |
+
):
|
| 1395 |
+
"""
|
| 1396 |
+
Initialize the mixture dataset.
|
| 1397 |
+
|
| 1398 |
+
Args:
|
| 1399 |
+
data_mixture (list[tuple[LeRobotSingleDataset, float]]): Datasets and their corresponding weights.
|
| 1400 |
+
mode (str): If "train", __getitem__ will return different samples every epoch; if "val" or "test", __getitem__ will return the same sample every epoch.
|
| 1401 |
+
balance_dataset_weights (bool): If True, the weight of dataset will be multiplied by the total trajectory length of each dataset.
|
| 1402 |
+
balance_trajectory_weights (bool): If True, sample trajectories within a dataset weighted by their length; otherwise, use equal weighting.
|
| 1403 |
+
seed (int): Random seed for sampling.
|
| 1404 |
+
"""
|
| 1405 |
+
datasets: list[LeRobotSingleDataset] = []
|
| 1406 |
+
dataset_sampling_weights: list[float] = []
|
| 1407 |
+
for dataset, weight in data_mixture:
|
| 1408 |
+
# Check if dataset is valid and has data
|
| 1409 |
+
if len(dataset) == 0:
|
| 1410 |
+
print(f"Warning: Skipping empty dataset {dataset.dataset_name}")
|
| 1411 |
+
continue
|
| 1412 |
+
datasets.append(dataset)
|
| 1413 |
+
dataset_sampling_weights.append(weight)
|
| 1414 |
+
|
| 1415 |
+
if len(datasets) == 0:
|
| 1416 |
+
raise ValueError("No valid datasets found in the mixture. All datasets are empty.")
|
| 1417 |
+
|
| 1418 |
+
self.datasets = datasets
|
| 1419 |
+
self.balance_dataset_weights = balance_dataset_weights
|
| 1420 |
+
self.balance_trajectory_weights = balance_trajectory_weights
|
| 1421 |
+
self.seed = seed
|
| 1422 |
+
self.mode = mode
|
| 1423 |
+
|
| 1424 |
+
# Set properties for sampling
|
| 1425 |
+
|
| 1426 |
+
# 1. Dataset lengths
|
| 1427 |
+
self._dataset_lengths = np.array([len(dataset) for dataset in self.datasets])
|
| 1428 |
+
print(f"Dataset lengths: {self._dataset_lengths}")
|
| 1429 |
+
|
| 1430 |
+
# 2. Dataset sampling weights
|
| 1431 |
+
self._dataset_sampling_weights = np.array(dataset_sampling_weights)
|
| 1432 |
+
|
| 1433 |
+
if self.balance_dataset_weights:
|
| 1434 |
+
self._dataset_sampling_weights *= self._dataset_lengths
|
| 1435 |
+
|
| 1436 |
+
# Check for zero or negative weights before normalization
|
| 1437 |
+
if np.any(self._dataset_sampling_weights <= 0):
|
| 1438 |
+
print(f"Warning: Found zero or negative sampling weights: {self._dataset_sampling_weights}")
|
| 1439 |
+
# Set minimum weight to prevent division issues
|
| 1440 |
+
self._dataset_sampling_weights = np.maximum(self._dataset_sampling_weights, 1e-8)
|
| 1441 |
+
|
| 1442 |
+
# Normalize weights
|
| 1443 |
+
weights_sum = self._dataset_sampling_weights.sum()
|
| 1444 |
+
if weights_sum == 0 or np.isnan(weights_sum):
|
| 1445 |
+
print(f"Error: Invalid weights sum: {weights_sum}")
|
| 1446 |
+
# Fallback to equal weights
|
| 1447 |
+
self._dataset_sampling_weights = np.ones(len(self.datasets)) / len(self.datasets)
|
| 1448 |
+
print(f"Fallback to equal weights")
|
| 1449 |
+
else:
|
| 1450 |
+
self._dataset_sampling_weights /= weights_sum
|
| 1451 |
+
|
| 1452 |
+
# 3. Trajectory sampling weights
|
| 1453 |
+
self._trajectory_sampling_weights: list[np.ndarray] = []
|
| 1454 |
+
for i, dataset in enumerate(self.datasets):
|
| 1455 |
+
trajectory_sampling_weights = np.ones(len(dataset.trajectory_lengths))
|
| 1456 |
+
if self.balance_trajectory_weights:
|
| 1457 |
+
trajectory_sampling_weights *= dataset.trajectory_lengths
|
| 1458 |
+
|
| 1459 |
+
# Check for zero or negative weights before normalization
|
| 1460 |
+
if np.any(trajectory_sampling_weights <= 0):
|
| 1461 |
+
print(f"Warning: Dataset {i} has zero or negative trajectory weights")
|
| 1462 |
+
trajectory_sampling_weights = np.maximum(trajectory_sampling_weights, 1e-8)
|
| 1463 |
+
|
| 1464 |
+
# Normalize weights
|
| 1465 |
+
weights_sum = trajectory_sampling_weights.sum()
|
| 1466 |
+
if weights_sum == 0 or np.isnan(weights_sum):
|
| 1467 |
+
print(f"Error: Dataset {i} has invalid trajectory weights sum: {weights_sum}")
|
| 1468 |
+
# Fallback to equal weights
|
| 1469 |
+
trajectory_sampling_weights = np.ones(len(dataset.trajectory_lengths)) / len(dataset.trajectory_lengths)
|
| 1470 |
+
else:
|
| 1471 |
+
trajectory_sampling_weights /= weights_sum
|
| 1472 |
+
|
| 1473 |
+
self._trajectory_sampling_weights.append(trajectory_sampling_weights)
|
| 1474 |
+
|
| 1475 |
+
# 4. Primary dataset indices
|
| 1476 |
+
self._primary_dataset_indices = np.array(dataset_sampling_weights) == 1.0
|
| 1477 |
+
if not np.any(self._primary_dataset_indices):
|
| 1478 |
+
print(f"Warning: No dataset with weight 1.0 found. Original weights: {dataset_sampling_weights}")
|
| 1479 |
+
# Fallback: use the dataset(s) with maximum weight as primary
|
| 1480 |
+
max_weight = max(dataset_sampling_weights)
|
| 1481 |
+
self._primary_dataset_indices = np.array(dataset_sampling_weights) == max_weight
|
| 1482 |
+
print(f"Using datasets with maximum weight {max_weight} as primary: {self._primary_dataset_indices}")
|
| 1483 |
+
|
| 1484 |
+
if not np.any(self._primary_dataset_indices):
|
| 1485 |
+
# This should never happen, but just in case
|
| 1486 |
+
print("Error: Still no primary dataset found. Using first dataset as primary.")
|
| 1487 |
+
self._primary_dataset_indices = np.zeros(len(self.datasets), dtype=bool)
|
| 1488 |
+
self._primary_dataset_indices[0] = True
|
| 1489 |
+
|
| 1490 |
+
# Set the epoch and sample the first epoch
|
| 1491 |
+
self.set_epoch(0)
|
| 1492 |
+
|
| 1493 |
+
self.update_metadata(metadata_config)
|
| 1494 |
+
|
| 1495 |
+
@property
|
| 1496 |
+
def dataset_lengths(self) -> np.ndarray:
|
| 1497 |
+
"""The lengths of each dataset."""
|
| 1498 |
+
return self._dataset_lengths
|
| 1499 |
+
|
| 1500 |
+
@property
|
| 1501 |
+
def dataset_sampling_weights(self) -> np.ndarray:
|
| 1502 |
+
"""The sampling weights for each dataset."""
|
| 1503 |
+
return self._dataset_sampling_weights
|
| 1504 |
+
|
| 1505 |
+
@property
|
| 1506 |
+
def trajectory_sampling_weights(self) -> list[np.ndarray]:
|
| 1507 |
+
"""The sampling weights for each trajectory in each dataset."""
|
| 1508 |
+
return self._trajectory_sampling_weights
|
| 1509 |
+
|
| 1510 |
+
@property
|
| 1511 |
+
def primary_dataset_indices(self) -> np.ndarray:
|
| 1512 |
+
"""The indices of the primary datasets."""
|
| 1513 |
+
return self._primary_dataset_indices
|
| 1514 |
+
|
| 1515 |
+
def __str__(self) -> str:
|
| 1516 |
+
dataset_descriptions = []
|
| 1517 |
+
for dataset, weight in zip(self.datasets, self.dataset_sampling_weights):
|
| 1518 |
+
dataset_description = {
|
| 1519 |
+
"Dataset": str(dataset),
|
| 1520 |
+
"Sampling weight": float(weight),
|
| 1521 |
+
}
|
| 1522 |
+
dataset_descriptions.append(dataset_description)
|
| 1523 |
+
return json.dumps({"Mixture dataset": dataset_descriptions}, indent=2)
|
| 1524 |
+
|
| 1525 |
+
def set_epoch(self, epoch: int):
|
| 1526 |
+
"""Set the epoch for the dataset.
|
| 1527 |
+
|
| 1528 |
+
Args:
|
| 1529 |
+
epoch (int): The epoch to set.
|
| 1530 |
+
"""
|
| 1531 |
+
self.epoch = epoch
|
| 1532 |
+
# self.sampled_steps = self.sample_epoch()
|
| 1533 |
+
|
| 1534 |
+
def sample_step(self, index: int) -> tuple[LeRobotSingleDataset, int, int]:
|
| 1535 |
+
"""Sample a single step from the dataset."""
|
| 1536 |
+
# return self.sampled_steps[index]
|
| 1537 |
+
|
| 1538 |
+
# Set seed
|
| 1539 |
+
seed = index if self.mode != "train" else safe_hash((self.epoch, index, self.seed))
|
| 1540 |
+
rng = np.random.default_rng(seed)
|
| 1541 |
+
|
| 1542 |
+
# Sample dataset
|
| 1543 |
+
dataset_index = rng.choice(len(self.datasets), p=self.dataset_sampling_weights)
|
| 1544 |
+
dataset = self.datasets[dataset_index]
|
| 1545 |
+
|
| 1546 |
+
# Sample trajectory
|
| 1547 |
+
# trajectory_index = rng.choice(
|
| 1548 |
+
# len(dataset.trajectory_ids), p=self.trajectory_sampling_weights[dataset_index]
|
| 1549 |
+
# )
|
| 1550 |
+
# trajectory_id = dataset.trajectory_ids[trajectory_index]
|
| 1551 |
+
|
| 1552 |
+
# # Sample step
|
| 1553 |
+
# base_index = rng.choice(dataset.trajectory_lengths[trajectory_index])
|
| 1554 |
+
# return dataset, trajectory_id, base_index
|
| 1555 |
+
single_step_index = rng.choice(len(dataset.all_steps))
|
| 1556 |
+
trajectory_id, base_index = dataset.all_steps[single_step_index]
|
| 1557 |
+
return dataset, trajectory_id, base_index
|
| 1558 |
+
|
| 1559 |
+
def sample_step_with_shuffle_is_true(self, index: int) -> tuple[LeRobotSingleDataset, int, int]:
|
| 1560 |
+
"""Sample a single step from the dataset."""
|
| 1561 |
+
'''only use this when DataLoader is initialized with shuffle=True'''
|
| 1562 |
+
print(index)
|
| 1563 |
+
# return self.sampled_steps[index]
|
| 1564 |
+
|
| 1565 |
+
# Set seed
|
| 1566 |
+
# seed = index if self.mode != "train" else safe_hash((self.epoch, index, self.seed))
|
| 1567 |
+
# rng = np.random.default_rng(seed)
|
| 1568 |
+
|
| 1569 |
+
# Sample dataset
|
| 1570 |
+
# dataset_index = rng.choice(len(self.datasets), p=self.dataset_sampling_weights)
|
| 1571 |
+
# dataset = self.datasets[dataset_index]
|
| 1572 |
+
dataset = self.datasets[0] #FIXME
|
| 1573 |
+
|
| 1574 |
+
# Sample trajectory
|
| 1575 |
+
# trajectory_index = rng.choice(
|
| 1576 |
+
# len(dataset.trajectory_ids), p=self.trajectory_sampling_weights[dataset_index]
|
| 1577 |
+
# )
|
| 1578 |
+
# trajectory_id = dataset.trajectory_ids[trajectory_index]
|
| 1579 |
+
|
| 1580 |
+
# # Sample step
|
| 1581 |
+
# base_index = rng.choice(dataset.trajectory_lengths[trajectory_index])
|
| 1582 |
+
# return dataset, trajectory_id, base_index
|
| 1583 |
+
# no random, follow the order of the dataset
|
| 1584 |
+
# single_step_index = rng.choice(len(dataset.all_steps))
|
| 1585 |
+
# trajectory_id, base_index = dataset.all_steps[single_step_index]
|
| 1586 |
+
|
| 1587 |
+
trajectory_id, base_index = dataset.all_steps[index]
|
| 1588 |
+
return dataset, trajectory_id, base_index
|
| 1589 |
+
|
| 1590 |
+
def __getitem__(self, index: int) -> dict:
|
| 1591 |
+
"""Get the data for a single trajectory and start index.
|
| 1592 |
+
|
| 1593 |
+
Args:
|
| 1594 |
+
index (int): The index of the trajectory to get.
|
| 1595 |
+
|
| 1596 |
+
Returns:
|
| 1597 |
+
dict: The data for the trajectory and start index.
|
| 1598 |
+
"""
|
| 1599 |
+
max_retries = 10
|
| 1600 |
+
last_exception = None
|
| 1601 |
+
|
| 1602 |
+
for attempt in range(max_retries):
|
| 1603 |
+
try:
|
| 1604 |
+
dataset, trajectory_name, step = self.sample_step(index)
|
| 1605 |
+
data = dataset.transforms(dataset.get_step_data(trajectory_name, step))
|
| 1606 |
+
|
| 1607 |
+
# Process all video keys dynamically
|
| 1608 |
+
images = []
|
| 1609 |
+
for video_key in dataset.modality_keys["video"]:
|
| 1610 |
+
image = data[video_key][0]
|
| 1611 |
+
|
| 1612 |
+
# Apply image cropping if enabled and the video key is base_view
|
| 1613 |
+
# Note: crop_obs_camera functionality has been removed
|
| 1614 |
+
|
| 1615 |
+
image = Image.fromarray(image).resize((224, 224))
|
| 1616 |
+
# # use time stamp as filename
|
| 1617 |
+
# import time
|
| 1618 |
+
# filename = f"/hfm/boqian/storage/tmp/image_{int(time.time())}_{index}.png"
|
| 1619 |
+
# image.save(filename)
|
| 1620 |
+
images.append(image)
|
| 1621 |
+
|
| 1622 |
+
# Get language and action data
|
| 1623 |
+
language = data[dataset.modality_keys["language"][0]][0]
|
| 1624 |
+
action = []
|
| 1625 |
+
for action_key in dataset.modality_keys["action"]:
|
| 1626 |
+
action.append(data[action_key])
|
| 1627 |
+
action = np.concatenate(action, axis=1).astype(np.float16)
|
| 1628 |
+
|
| 1629 |
+
return dict(action=action, image=images, lang=language)
|
| 1630 |
+
|
| 1631 |
+
except Exception as e:
|
| 1632 |
+
last_exception = e
|
| 1633 |
+
if attempt < max_retries - 1:
|
| 1634 |
+
# Log the error but continue trying
|
| 1635 |
+
print(f"Attempt {attempt + 1}/{max_retries} failed for index {index}: {e}")
|
| 1636 |
+
print(f"Retrying with new sample...")
|
| 1637 |
+
# For retry, we can use a slightly different index to get a new sample
|
| 1638 |
+
# This helps avoid getting stuck on the same problematic sample
|
| 1639 |
+
index = (index + 1) % len(self)
|
| 1640 |
+
else:
|
| 1641 |
+
# All retries exhausted
|
| 1642 |
+
print(f"All {max_retries} attempts failed for index {index}")
|
| 1643 |
+
print(f"Last error: {last_exception}")
|
| 1644 |
+
# Return a dummy sample or re-raise the exception
|
| 1645 |
+
raise last_exception
|
| 1646 |
+
|
| 1647 |
+
def __len__(self) -> int:
|
| 1648 |
+
"""Get the length of a single epoch in the mixture.
|
| 1649 |
+
|
| 1650 |
+
Returns:
|
| 1651 |
+
int: The length of a single epoch in the mixture.
|
| 1652 |
+
"""
|
| 1653 |
+
# Check for potential issues
|
| 1654 |
+
if len(self.datasets) == 0:
|
| 1655 |
+
return 0
|
| 1656 |
+
|
| 1657 |
+
# Check if any dataset lengths are 0 or NaN
|
| 1658 |
+
if np.any(self.dataset_lengths == 0) or np.any(np.isnan(self.dataset_lengths)):
|
| 1659 |
+
print(f"Warning: Found zero or NaN dataset lengths: {self.dataset_lengths}")
|
| 1660 |
+
# Filter out zero/NaN length datasets
|
| 1661 |
+
valid_indices = (self.dataset_lengths > 0) & (~np.isnan(self.dataset_lengths))
|
| 1662 |
+
if not np.any(valid_indices):
|
| 1663 |
+
print("Error: All datasets have zero or NaN length")
|
| 1664 |
+
return 0
|
| 1665 |
+
else:
|
| 1666 |
+
valid_indices = np.ones(len(self.datasets), dtype=bool)
|
| 1667 |
+
|
| 1668 |
+
# Check if any sampling weights are 0 or NaN
|
| 1669 |
+
if np.any(self.dataset_sampling_weights == 0) or np.any(np.isnan(self.dataset_sampling_weights)):
|
| 1670 |
+
print(f"Warning: Found zero or NaN sampling weights: {self.dataset_sampling_weights}")
|
| 1671 |
+
# Use only valid weights
|
| 1672 |
+
valid_weights = (self.dataset_sampling_weights > 0) & (~np.isnan(self.dataset_sampling_weights))
|
| 1673 |
+
valid_indices = valid_indices & valid_weights
|
| 1674 |
+
if not np.any(valid_indices):
|
| 1675 |
+
print("Error: All sampling weights are zero or NaN")
|
| 1676 |
+
return 0
|
| 1677 |
+
|
| 1678 |
+
# Check primary dataset indices
|
| 1679 |
+
primary_and_valid = self.primary_dataset_indices & valid_indices
|
| 1680 |
+
if not np.any(primary_and_valid):
|
| 1681 |
+
print(f"Warning: No valid primary datasets found. Primary indices: {self.primary_dataset_indices}, Valid indices: {valid_indices}")
|
| 1682 |
+
# Fallback: use the largest valid dataset
|
| 1683 |
+
if np.any(valid_indices):
|
| 1684 |
+
max_length = self.dataset_lengths[valid_indices].max()
|
| 1685 |
+
print(f"Fallback: Using maximum dataset length: {max_length}")
|
| 1686 |
+
return int(max_length)
|
| 1687 |
+
else:
|
| 1688 |
+
return 0
|
| 1689 |
+
|
| 1690 |
+
# Calculate the ratio and get max
|
| 1691 |
+
ratios = (self.dataset_lengths / self.dataset_sampling_weights)[primary_and_valid]
|
| 1692 |
+
|
| 1693 |
+
# Check for NaN or inf in ratios
|
| 1694 |
+
if np.any(np.isnan(ratios)) or np.any(np.isinf(ratios)):
|
| 1695 |
+
print(f"Warning: Found NaN or inf in ratios: {ratios}")
|
| 1696 |
+
print(f"Dataset lengths: {self.dataset_lengths[primary_and_valid]}")
|
| 1697 |
+
print(f"Sampling weights: {self.dataset_sampling_weights[primary_and_valid]}")
|
| 1698 |
+
# Filter out invalid ratios
|
| 1699 |
+
valid_ratios = ratios[~np.isnan(ratios) & ~np.isinf(ratios)]
|
| 1700 |
+
if len(valid_ratios) == 0:
|
| 1701 |
+
print("Error: All ratios are NaN or inf")
|
| 1702 |
+
return 0
|
| 1703 |
+
max_ratio = valid_ratios.max()
|
| 1704 |
+
else:
|
| 1705 |
+
max_ratio = ratios.max()
|
| 1706 |
+
|
| 1707 |
+
result = int(max_ratio)
|
| 1708 |
+
if result == 0:
|
| 1709 |
+
print(f"Warning: Dataset mixture length is 0")
|
| 1710 |
+
return result
|
| 1711 |
+
|
| 1712 |
+
@staticmethod
|
| 1713 |
+
def compute_overall_statistics(
|
| 1714 |
+
per_task_stats: list[dict[str, dict[str, list[float] | np.ndarray]]],
|
| 1715 |
+
dataset_sampling_weights: list[float] | np.ndarray,
|
| 1716 |
+
percentile_mixing_method: str = "weighted_average",
|
| 1717 |
+
) -> dict[str, dict[str, list[float]]]:
|
| 1718 |
+
"""
|
| 1719 |
+
Computes overall statistics from per-task statistics using dataset sample weights.
|
| 1720 |
+
|
| 1721 |
+
Args:
|
| 1722 |
+
per_task_stats: List of per-task statistics.
|
| 1723 |
+
Example format of one element in the per-task statistics list:
|
| 1724 |
+
{
|
| 1725 |
+
"state.gripper": {
|
| 1726 |
+
"min": [...],
|
| 1727 |
+
"max": [...],
|
| 1728 |
+
"mean": [...],
|
| 1729 |
+
"std": [...],
|
| 1730 |
+
"q01": [...],
|
| 1731 |
+
"q99": [...],
|
| 1732 |
+
},
|
| 1733 |
+
...
|
| 1734 |
+
}
|
| 1735 |
+
dataset_sampling_weights: List of sample weights for each task.
|
| 1736 |
+
percentile_mixing_method: The method to mix the percentiles, either "weighted_average" or "weighted_std".
|
| 1737 |
+
|
| 1738 |
+
Returns:
|
| 1739 |
+
A dict of overall statistics per modality.
|
| 1740 |
+
"""
|
| 1741 |
+
# Normalize the sample weights to sum to 1
|
| 1742 |
+
dataset_sampling_weights = np.array(dataset_sampling_weights)
|
| 1743 |
+
normalized_weights = dataset_sampling_weights / dataset_sampling_weights.sum()
|
| 1744 |
+
|
| 1745 |
+
# Initialize overall statistics dict
|
| 1746 |
+
overall_stats: dict[str, dict[str, list[float]]] = {}
|
| 1747 |
+
|
| 1748 |
+
# Get the list of modality keys
|
| 1749 |
+
modality_keys = per_task_stats[0].keys()
|
| 1750 |
+
|
| 1751 |
+
for modality in modality_keys:
|
| 1752 |
+
# Number of dimensions (assuming consistent across tasks)
|
| 1753 |
+
num_dims = len(per_task_stats[0][modality]["mean"])
|
| 1754 |
+
|
| 1755 |
+
# Initialize accumulators for means and variances
|
| 1756 |
+
weighted_means = np.zeros(num_dims)
|
| 1757 |
+
weighted_squares = np.zeros(num_dims)
|
| 1758 |
+
|
| 1759 |
+
# Collect min, max, q01, q99 from all tasks
|
| 1760 |
+
min_list = []
|
| 1761 |
+
max_list = []
|
| 1762 |
+
q01_list = []
|
| 1763 |
+
q99_list = []
|
| 1764 |
+
|
| 1765 |
+
for task_idx, task_stats in enumerate(per_task_stats):
|
| 1766 |
+
w_i = normalized_weights[task_idx]
|
| 1767 |
+
stats = task_stats[modality]
|
| 1768 |
+
means = np.array(stats["mean"])
|
| 1769 |
+
stds = np.array(stats["std"])
|
| 1770 |
+
|
| 1771 |
+
# Update weighted sums for mean and variance
|
| 1772 |
+
weighted_means += w_i * means
|
| 1773 |
+
weighted_squares += w_i * (stds**2 + means**2)
|
| 1774 |
+
|
| 1775 |
+
# Collect min, max, q01, q99
|
| 1776 |
+
min_list.append(stats["min"])
|
| 1777 |
+
max_list.append(stats["max"])
|
| 1778 |
+
q01_list.append(stats["q01"])
|
| 1779 |
+
q99_list.append(stats["q99"])
|
| 1780 |
+
|
| 1781 |
+
# Compute overall mean
|
| 1782 |
+
overall_mean = weighted_means.tolist()
|
| 1783 |
+
|
| 1784 |
+
# Compute overall variance and std deviation
|
| 1785 |
+
overall_variance = weighted_squares - weighted_means**2
|
| 1786 |
+
overall_std = np.sqrt(overall_variance).tolist()
|
| 1787 |
+
|
| 1788 |
+
# Compute overall min and max per dimension
|
| 1789 |
+
overall_min = np.min(np.array(min_list), axis=0).tolist()
|
| 1790 |
+
overall_max = np.max(np.array(max_list), axis=0).tolist()
|
| 1791 |
+
|
| 1792 |
+
# Compute overall q01 and q99 per dimension
|
| 1793 |
+
# Use weighted average of per-task quantiles
|
| 1794 |
+
q01_array = np.array(q01_list)
|
| 1795 |
+
q99_array = np.array(q99_list)
|
| 1796 |
+
if percentile_mixing_method == "weighted_average":
|
| 1797 |
+
weighted_q01 = np.average(q01_array, axis=0, weights=normalized_weights).tolist()
|
| 1798 |
+
weighted_q99 = np.average(q99_array, axis=0, weights=normalized_weights).tolist()
|
| 1799 |
+
# std_q01 = np.std(q01_array, axis=0).tolist()
|
| 1800 |
+
# std_q99 = np.std(q99_array, axis=0).tolist()
|
| 1801 |
+
# print(modality)
|
| 1802 |
+
# print(f"{std_q01=}, {std_q99=}")
|
| 1803 |
+
# print(f"{weighted_q01=}, {weighted_q99=}")
|
| 1804 |
+
elif percentile_mixing_method == "min_max":
|
| 1805 |
+
weighted_q01 = np.min(q01_array, axis=0).tolist()
|
| 1806 |
+
weighted_q99 = np.max(q99_array, axis=0).tolist()
|
| 1807 |
+
else:
|
| 1808 |
+
raise ValueError(f"Invalid percentile mixing method: {percentile_mixing_method}")
|
| 1809 |
+
|
| 1810 |
+
# Store the overall statistics for the modality
|
| 1811 |
+
overall_stats[modality] = {
|
| 1812 |
+
"min": overall_min,
|
| 1813 |
+
"max": overall_max,
|
| 1814 |
+
"mean": overall_mean,
|
| 1815 |
+
"std": overall_std,
|
| 1816 |
+
"q01": weighted_q01,
|
| 1817 |
+
"q99": weighted_q99,
|
| 1818 |
+
}
|
| 1819 |
+
|
| 1820 |
+
return overall_stats
|
| 1821 |
+
|
| 1822 |
+
@staticmethod
|
| 1823 |
+
def merge_metadata(
|
| 1824 |
+
metadatas: list[DatasetMetadata],
|
| 1825 |
+
dataset_sampling_weights: list[float],
|
| 1826 |
+
percentile_mixing_method: str,
|
| 1827 |
+
) -> DatasetMetadata:
|
| 1828 |
+
"""Merge multiple metadata into one."""
|
| 1829 |
+
# Convert to dicts
|
| 1830 |
+
metadata_dicts = [metadata.model_dump(mode="json") for metadata in metadatas]
|
| 1831 |
+
# Create a new metadata dict
|
| 1832 |
+
merged_metadata = {}
|
| 1833 |
+
|
| 1834 |
+
# Check all metadata have the same embodiment tag
|
| 1835 |
+
assert all(
|
| 1836 |
+
metadata.embodiment_tag == metadatas[0].embodiment_tag for metadata in metadatas
|
| 1837 |
+
), "All metadata must have the same embodiment tag"
|
| 1838 |
+
merged_metadata["embodiment_tag"] = metadatas[0].embodiment_tag
|
| 1839 |
+
|
| 1840 |
+
# Merge the dataset statistics
|
| 1841 |
+
dataset_statistics = {}
|
| 1842 |
+
dataset_statistics["state"] = LeRobotMixtureDataset.compute_overall_statistics(
|
| 1843 |
+
per_task_stats=[m["statistics"]["state"] for m in metadata_dicts],
|
| 1844 |
+
dataset_sampling_weights=dataset_sampling_weights,
|
| 1845 |
+
percentile_mixing_method=percentile_mixing_method,
|
| 1846 |
+
)
|
| 1847 |
+
dataset_statistics["action"] = LeRobotMixtureDataset.compute_overall_statistics(
|
| 1848 |
+
per_task_stats=[m["statistics"]["action"] for m in metadata_dicts],
|
| 1849 |
+
dataset_sampling_weights=dataset_sampling_weights,
|
| 1850 |
+
percentile_mixing_method=percentile_mixing_method,
|
| 1851 |
+
)
|
| 1852 |
+
merged_metadata["statistics"] = dataset_statistics
|
| 1853 |
+
|
| 1854 |
+
# Merge the modality configs
|
| 1855 |
+
modality_configs = defaultdict(set)
|
| 1856 |
+
for metadata in metadata_dicts:
|
| 1857 |
+
for modality, configs in metadata["modalities"].items():
|
| 1858 |
+
modality_configs[modality].add(json.dumps(configs))
|
| 1859 |
+
merged_metadata["modalities"] = {}
|
| 1860 |
+
for modality, configs in modality_configs.items():
|
| 1861 |
+
# Check that all modality configs correspond to the same tag matches
|
| 1862 |
+
assert (
|
| 1863 |
+
len(configs) == 1
|
| 1864 |
+
), f"Multiple modality configs for modality {modality}: {list(configs)}"
|
| 1865 |
+
merged_metadata["modalities"][modality] = json.loads(configs.pop())
|
| 1866 |
+
|
| 1867 |
+
return DatasetMetadata.model_validate(merged_metadata)
|
| 1868 |
+
|
| 1869 |
+
def update_metadata(self, metadata_config: dict, cached_statistics_path: Path | str | None = None) -> None:
|
| 1870 |
+
"""
|
| 1871 |
+
Merge multiple metadatas into one and set the transforms with the merged metadata.
|
| 1872 |
+
|
| 1873 |
+
Args:
|
| 1874 |
+
metadata_config (dict): Configuration for the metadata.
|
| 1875 |
+
"percentile_mixing_method": The method to mix the percentiles, either "weighted_average" or "min_max".
|
| 1876 |
+
weighted_average: Use the weighted average of the percentiles using the weight used in sampling the datasets.
|
| 1877 |
+
min_max: Use the min of the 1st percentile and max of the 99th percentile.
|
| 1878 |
+
"""
|
| 1879 |
+
# If cached path is provided, try to load and apply
|
| 1880 |
+
if cached_statistics_path is not None:
|
| 1881 |
+
try:
|
| 1882 |
+
cached_stats = self.load_merged_statistics(cached_statistics_path)
|
| 1883 |
+
self.apply_cached_statistics(cached_stats)
|
| 1884 |
+
return
|
| 1885 |
+
except (FileNotFoundError, KeyError, ValidationError) as e:
|
| 1886 |
+
print(f"Failed to load cached statistics: {e}")
|
| 1887 |
+
print("Falling back to computing statistics from scratch...")
|
| 1888 |
+
|
| 1889 |
+
self.tag = EmbodimentTag.NEW_EMBODIMENT.value
|
| 1890 |
+
self.merged_metadata: dict[str, DatasetMetadata] = {}
|
| 1891 |
+
# Group metadata by tag
|
| 1892 |
+
all_metadatas: dict[str, list[DatasetMetadata]] = {}
|
| 1893 |
+
for dataset in self.datasets:
|
| 1894 |
+
if dataset.tag not in all_metadatas:
|
| 1895 |
+
all_metadatas[dataset.tag] = []
|
| 1896 |
+
all_metadatas[dataset.tag].append(dataset.metadata)
|
| 1897 |
+
for tag, metadatas in all_metadatas.items():
|
| 1898 |
+
self.merged_metadata[tag] = self.merge_metadata(
|
| 1899 |
+
metadatas=metadatas,
|
| 1900 |
+
dataset_sampling_weights=self.dataset_sampling_weights.tolist(),
|
| 1901 |
+
percentile_mixing_method=metadata_config["percentile_mixing_method"],
|
| 1902 |
+
)
|
| 1903 |
+
for dataset in self.datasets:
|
| 1904 |
+
dataset.set_transforms_metadata(self.merged_metadata[dataset.tag])
|
| 1905 |
+
|
| 1906 |
+
def save_dataset_statistics(self, save_path: Path | str, format: str = "json") -> None:
|
| 1907 |
+
"""
|
| 1908 |
+
Save merged dataset statistics to specified path in the required format.
|
| 1909 |
+
Only includes statistics for keys that are actually used in the datasets.
|
| 1910 |
+
Gripper-related keys will be placed at the end.
|
| 1911 |
+
|
| 1912 |
+
Args:
|
| 1913 |
+
save_path (Path | str): Path to save the statistics file
|
| 1914 |
+
format (str): Save format, currently only supports "json"
|
| 1915 |
+
"""
|
| 1916 |
+
save_path = Path(save_path)
|
| 1917 |
+
save_path.parent.mkdir(parents=True, exist_ok=True)
|
| 1918 |
+
|
| 1919 |
+
# Build the data structure to save
|
| 1920 |
+
statistics_data = {}
|
| 1921 |
+
|
| 1922 |
+
# Collect actually used keys from all datasets
|
| 1923 |
+
all_used_action_keys = set()
|
| 1924 |
+
all_used_state_keys = set()
|
| 1925 |
+
|
| 1926 |
+
for dataset in self.datasets:
|
| 1927 |
+
used_action_keys, used_state_keys = get_used_modality_keys(dataset.modality_keys)
|
| 1928 |
+
all_used_action_keys.update(used_action_keys)
|
| 1929 |
+
all_used_state_keys.update(used_state_keys)
|
| 1930 |
+
|
| 1931 |
+
# Organize statistics by tag
|
| 1932 |
+
for tag, merged_metadata in self.merged_metadata.items():
|
| 1933 |
+
tag_stats = {}
|
| 1934 |
+
|
| 1935 |
+
# Process action statistics
|
| 1936 |
+
if hasattr(merged_metadata.statistics, 'action') and merged_metadata.statistics.action:
|
| 1937 |
+
action_stats = merged_metadata.statistics.action
|
| 1938 |
+
|
| 1939 |
+
# Filter and reorder keys
|
| 1940 |
+
non_gripper_keys = []
|
| 1941 |
+
gripper_keys = []
|
| 1942 |
+
|
| 1943 |
+
for key in action_stats.keys():
|
| 1944 |
+
if key in all_used_action_keys:
|
| 1945 |
+
if "gripper" in key.lower():
|
| 1946 |
+
gripper_keys.append(key)
|
| 1947 |
+
else:
|
| 1948 |
+
non_gripper_keys.append(key)
|
| 1949 |
+
|
| 1950 |
+
reordered_keys = non_gripper_keys + gripper_keys
|
| 1951 |
+
|
| 1952 |
+
filtered_action_stats = {}
|
| 1953 |
+
for key in reordered_keys:
|
| 1954 |
+
filtered_action_stats[key] = action_stats[key]
|
| 1955 |
+
|
| 1956 |
+
if filtered_action_stats:
|
| 1957 |
+
combined_action_stats = combine_modality_stats(filtered_action_stats)
|
| 1958 |
+
|
| 1959 |
+
mask = generate_action_mask_for_used_keys(
|
| 1960 |
+
merged_metadata.modalities.action, filtered_action_stats.keys()
|
| 1961 |
+
)
|
| 1962 |
+
combined_action_stats["mask"] = mask
|
| 1963 |
+
|
| 1964 |
+
tag_stats["action"] = combined_action_stats
|
| 1965 |
+
|
| 1966 |
+
# Process state statistics
|
| 1967 |
+
if hasattr(merged_metadata.statistics, 'state') and merged_metadata.statistics.state:
|
| 1968 |
+
state_stats = merged_metadata.statistics.state
|
| 1969 |
+
|
| 1970 |
+
# Filter and reorder keys
|
| 1971 |
+
non_gripper_keys = []
|
| 1972 |
+
gripper_keys = []
|
| 1973 |
+
|
| 1974 |
+
for key in state_stats.keys():
|
| 1975 |
+
if key in all_used_state_keys:
|
| 1976 |
+
if "gripper" in key.lower():
|
| 1977 |
+
gripper_keys.append(key)
|
| 1978 |
+
else:
|
| 1979 |
+
non_gripper_keys.append(key)
|
| 1980 |
+
|
| 1981 |
+
reordered_keys = non_gripper_keys + gripper_keys
|
| 1982 |
+
|
| 1983 |
+
filtered_state_stats = {}
|
| 1984 |
+
for key in reordered_keys:
|
| 1985 |
+
filtered_state_stats[key] = state_stats[key]
|
| 1986 |
+
|
| 1987 |
+
if filtered_state_stats:
|
| 1988 |
+
combined_state_stats = combine_modality_stats(filtered_state_stats)
|
| 1989 |
+
tag_stats["state"] = combined_state_stats
|
| 1990 |
+
|
| 1991 |
+
# Add dataset counts
|
| 1992 |
+
tag_stats.update(self._get_dataset_counts(tag))
|
| 1993 |
+
|
| 1994 |
+
statistics_data[tag] = tag_stats
|
| 1995 |
+
|
| 1996 |
+
# Save file
|
| 1997 |
+
if format.lower() == "json":
|
| 1998 |
+
if not str(save_path).endswith('.json'):
|
| 1999 |
+
save_path = save_path.with_suffix('.json')
|
| 2000 |
+
with open(save_path, 'w', encoding='utf-8') as f:
|
| 2001 |
+
json.dump(statistics_data, f, indent=2, ensure_ascii=False)
|
| 2002 |
+
else:
|
| 2003 |
+
raise ValueError(f"Unsupported format: {format}. Currently only 'json' is supported.")
|
| 2004 |
+
|
| 2005 |
+
print(f"Merged dataset statistics saved to: {save_path}")
|
| 2006 |
+
print(f"Used action keys (reordered): {list(all_used_action_keys)}")
|
| 2007 |
+
print(f"Used state keys (reordered): {list(all_used_state_keys)}")
|
| 2008 |
+
|
| 2009 |
+
def _combine_modality_stats(self, modality_stats: dict) -> dict:
|
| 2010 |
+
"""Backward compatibility wrapper."""
|
| 2011 |
+
return combine_modality_stats(modality_stats)
|
| 2012 |
+
|
| 2013 |
+
def _generate_action_mask_for_used_keys(self, action_modalities: dict, used_action_keys_ordered) -> list[bool]:
|
| 2014 |
+
"""Backward compatibility wrapper."""
|
| 2015 |
+
return generate_action_mask_for_used_keys(action_modalities, used_action_keys_ordered)
|
| 2016 |
+
|
| 2017 |
+
def _get_dataset_counts(self, tag: str) -> dict:
|
| 2018 |
+
"""
|
| 2019 |
+
Get dataset count information for specified tag.
|
| 2020 |
+
|
| 2021 |
+
Args:
|
| 2022 |
+
tag (str): embodiment tag
|
| 2023 |
+
|
| 2024 |
+
Returns:
|
| 2025 |
+
dict: Dictionary containing num_transitions and num_trajectories
|
| 2026 |
+
"""
|
| 2027 |
+
num_transitions = 0
|
| 2028 |
+
num_trajectories = 0
|
| 2029 |
+
|
| 2030 |
+
# Count dataset information belonging to this tag
|
| 2031 |
+
for dataset in self.datasets:
|
| 2032 |
+
if dataset.tag == tag:
|
| 2033 |
+
num_transitions += len(dataset)
|
| 2034 |
+
num_trajectories += len(dataset.trajectory_ids)
|
| 2035 |
+
|
| 2036 |
+
return {
|
| 2037 |
+
"num_transitions": num_transitions,
|
| 2038 |
+
"num_trajectories": num_trajectories
|
| 2039 |
+
}
|
| 2040 |
+
|
| 2041 |
+
@classmethod
|
| 2042 |
+
def load_merged_statistics(cls, load_path: Path | str) -> dict:
|
| 2043 |
+
"""
|
| 2044 |
+
Load merged dataset statistics from file.
|
| 2045 |
+
|
| 2046 |
+
Args:
|
| 2047 |
+
load_path (Path | str): Path to the statistics file
|
| 2048 |
+
|
| 2049 |
+
Returns:
|
| 2050 |
+
dict: Dictionary containing merged statistics
|
| 2051 |
+
"""
|
| 2052 |
+
load_path = Path(load_path)
|
| 2053 |
+
if not load_path.exists():
|
| 2054 |
+
raise FileNotFoundError(f"Statistics file not found: {load_path}")
|
| 2055 |
+
|
| 2056 |
+
if load_path.suffix.lower() == '.json':
|
| 2057 |
+
with open(load_path, 'r', encoding='utf-8') as f:
|
| 2058 |
+
return json.load(f)
|
| 2059 |
+
elif load_path.suffix.lower() == '.pkl':
|
| 2060 |
+
import pickle
|
| 2061 |
+
with open(load_path, 'rb') as f:
|
| 2062 |
+
return pickle.load(f)
|
| 2063 |
+
else:
|
| 2064 |
+
raise ValueError(f"Unsupported file format: {load_path.suffix}")
|
| 2065 |
+
|
| 2066 |
+
def apply_cached_statistics(self, cached_statistics: dict) -> None:
|
| 2067 |
+
"""
|
| 2068 |
+
Apply cached statistics to avoid recomputation.
|
| 2069 |
+
|
| 2070 |
+
Args:
|
| 2071 |
+
cached_statistics (dict): Statistics loaded from file
|
| 2072 |
+
"""
|
| 2073 |
+
# Validate that cached statistics match current datasets
|
| 2074 |
+
if "metadata" in cached_statistics:
|
| 2075 |
+
cached_dataset_names = set(cached_statistics["metadata"]["dataset_names"])
|
| 2076 |
+
current_dataset_names = set(dataset.dataset_name for dataset in self.datasets)
|
| 2077 |
+
|
| 2078 |
+
if cached_dataset_names != current_dataset_names:
|
| 2079 |
+
print("Warning: Cached statistics dataset names don't match current datasets.")
|
| 2080 |
+
print(f"Cached: {cached_dataset_names}")
|
| 2081 |
+
print(f"Current: {current_dataset_names}")
|
| 2082 |
+
return
|
| 2083 |
+
|
| 2084 |
+
# Apply cached statistics
|
| 2085 |
+
self.merged_metadata = {}
|
| 2086 |
+
for tag, stats_data in cached_statistics.items():
|
| 2087 |
+
if tag == "metadata": # Skip metadata field
|
| 2088 |
+
continue
|
| 2089 |
+
|
| 2090 |
+
# Convert back to DatasetMetadata format
|
| 2091 |
+
metadata_dict = {
|
| 2092 |
+
"embodiment_tag": tag,
|
| 2093 |
+
"statistics": {
|
| 2094 |
+
"action": {},
|
| 2095 |
+
"state": {}
|
| 2096 |
+
},
|
| 2097 |
+
"modalities": {}
|
| 2098 |
+
}
|
| 2099 |
+
|
| 2100 |
+
# Convert action statistics back
|
| 2101 |
+
if "action" in stats_data:
|
| 2102 |
+
action_data = stats_data["action"]
|
| 2103 |
+
# This is simplified - you may need to split back to sub-keys
|
| 2104 |
+
metadata_dict["statistics"]["action"] = action_data
|
| 2105 |
+
|
| 2106 |
+
# Convert state statistics back
|
| 2107 |
+
if "state" in stats_data:
|
| 2108 |
+
state_data = stats_data["state"]
|
| 2109 |
+
metadata_dict["statistics"]["state"] = state_data
|
| 2110 |
+
|
| 2111 |
+
self.merged_metadata[tag] = DatasetMetadata.model_validate(metadata_dict)
|
| 2112 |
+
|
| 2113 |
+
# Update transforms metadata for each dataset
|
| 2114 |
+
for dataset in self.datasets:
|
| 2115 |
+
if dataset.tag in self.merged_metadata:
|
| 2116 |
+
dataset.set_transforms_metadata(self.merged_metadata[dataset.tag])
|
| 2117 |
+
|
| 2118 |
+
print(f"Applied cached statistics for {len(self.merged_metadata)} embodiment tags.")
|