Add files using upload-large-folder tool
Browse files- .env.sample +16 -0
- .gitignore +60 -0
- .gitmodules +3 -0
- LICENSE +17 -0
- Psi0-训练阶段-state-action总览.md +216 -0
- README.md +532 -0
- cache_dance_video_omni_embeds.sh +13 -0
- cache_motion_example_video_omni_embeds.sh +22 -0
- cache_omni_hidden_states_v20.sh +78 -0
- cache_video_omni_embeds.py +996 -0
- cache_video_omni_embeds.sh +275 -0
- cache_video_omni_embeds_dense_stride1_direct_full.sh +523 -0
- cache_video_omni_embeds_dense_stride1_full.sh +63 -0
- cache_video_omni_embeds_full.sh +31 -0
- dataset/seed/g1/csv/220714/jog_backward_start_001__A025.csv +0 -0
- dataset/seed/g1/csv/220714/jog_backward_start_003__A025.csv +0 -0
- dataset/seed/g1/csv/230215/idle_crouch_right_loop_R_002__A194.csv +0 -0
- dataset/seed/g1/csv/230306/crouch_idle_004__A244.csv +0 -0
- dataset/seed/g1/csv/240327/jump_twice_R_001__A533.csv +0 -0
- dataset/seed/g1/csv/240327/jump_twice_R_002__A533.csv +0 -0
- dataset/seed/g1/csv/240327/jump_twice_R_003__A533.csv +0 -0
- dataset/seed/g1/csv/240327/touch_toes_standing_R_001__A533.csv +0 -0
- dataset/seed/metadata/seed_metadata_v004_100.csv +101 -0
- dataset/seed/metadata/seed_metadata_v004_100_move_name.jsonl +100 -0
- flake.lock +27 -0
- flake.nix +105 -0
- infer.sh +39 -0
- infer_prompt.sh +213 -0
- infer_prompt_distill.sh +104 -0
- infer_prompt_short.sh +157 -0
- infer_render.sh +64 -0
- infer_render_auto.sh +709 -0
- plan_0.md +374 -0
- posttrain-v1-psi0-all-7task-tune-vlm-direct-dense-actionhead-L.sh +313 -0
- posttrain-v1-psi0-all-7task-tune-vlm-direct-dense-actionhead-M.sh +313 -0
- posttrain-v1-psi0-all-7task-tune-vlm-direct-dense-actionhead-perfect-resume.sh +469 -0
- posttrain-v1-psi0-all-7task-tune-vlm-direct-dense-full.sh +421 -0
- posttrain-v1-psi0-all-7task-tune-vlm-direct-dense.sh +305 -0
- posttrain-v1-psi0-all-text-tune-vlm.sh +282 -0
- posttrain-v1-psi0-all-tune-vlm-direct-dense.sh +297 -0
- posttrain-v1-psi0-all-tune-vlm.sh +282 -0
- posttrain-v1-psi0-all.sh +281 -0
- posttrain-v1-psi0-all_resume.sh +285 -0
- posttrain-v1-psi0-audio-clip-tune-vlm.sh +282 -0
- posttrain-v1-psi0-motion-distill.sh +349 -0
- posttrain-v1-psi0-text-ckpt-continue-video-clip-tune-vlm-direct-dense.sh +305 -0
- posttrain-v1-psi0-text-ckpt-continue-video-clip-tune-vlm.sh +290 -0
- prebuild-multimodal-records-cache.sh +280 -0
- pyproject.toml +85 -0
- uv.lock +0 -0
.env.sample
ADDED
|
@@ -0,0 +1,16 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
HF_TOKEN=
|
| 2 |
+
WANDB_API_KEY=
|
| 3 |
+
WANDB_ENTITY=
|
| 4 |
+
PSI_HOME=/hfm
|
| 5 |
+
DATA_HOME=/hfm/data
|
| 6 |
+
HF_HOME=/hfm/cache
|
| 7 |
+
TORCH_HOME=/hfm/cache
|
| 8 |
+
UV_CACHE_DIR=/hfm/cache
|
| 9 |
+
HF_LEROBOT_HOME=/hfm/data/lerobot
|
| 10 |
+
OMP_NUM_THREADS=8
|
| 11 |
+
TOKENIZERS_PARALLELISM=false
|
| 12 |
+
DEEPSPEED_LOG_LEVEL=warning
|
| 13 |
+
CUDA_LAUNCH_BLOCKING=true
|
| 14 |
+
TF_CPP_MIN_LOG_LEVEL=3
|
| 15 |
+
AV_LOG_LEVEL=quiet
|
| 16 |
+
PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION=python
|
.gitignore
ADDED
|
@@ -0,0 +1,60 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Python-generated files
|
| 2 |
+
__pycache__/
|
| 3 |
+
*.py[oc]
|
| 4 |
+
build/
|
| 5 |
+
dist/
|
| 6 |
+
wheels/
|
| 7 |
+
*.egg-info
|
| 8 |
+
|
| 9 |
+
# Virtual environments
|
| 10 |
+
.venv*
|
| 11 |
+
|
| 12 |
+
/data/
|
| 13 |
+
cache
|
| 14 |
+
*.mp4
|
| 15 |
+
ml-egodex_dataset/ # soft link to the egodex dataset
|
| 16 |
+
wandb/
|
| 17 |
+
runs/
|
| 18 |
+
checkpoints/
|
| 19 |
+
log/
|
| 20 |
+
playgrounds/
|
| 21 |
+
TODO.md
|
| 22 |
+
*.zip
|
| 23 |
+
|
| 24 |
+
runs_old/
|
| 25 |
+
tmp_dir/
|
| 26 |
+
.runs*
|
| 27 |
+
|
| 28 |
+
|
| 29 |
+
.env
|
| 30 |
+
.env.local
|
| 31 |
+
|
| 32 |
+
_lerobot_build
|
| 33 |
+
.cache
|
| 34 |
+
.data
|
| 35 |
+
.logs
|
| 36 |
+
|
| 37 |
+
*.local.yaml
|
| 38 |
+
|
| 39 |
+
|
| 40 |
+
vq/
|
| 41 |
+
extern/vq_bet_official/
|
| 42 |
+
src/fast/egodex-*
|
| 43 |
+
*.pkl
|
| 44 |
+
!src/fast/egodex-rel-50w-1x48-v2048-s100
|
| 45 |
+
|
| 46 |
+
.deepspeed_env
|
| 47 |
+
*.bak
|
| 48 |
+
#uv.lock
|
| 49 |
+
.vscode/
|
| 50 |
+
.direnv/
|
| 51 |
+
third_party/
|
| 52 |
+
src/gr00t/.venv/
|
| 53 |
+
props/
|
| 54 |
+
textures/
|
| 55 |
+
test.py
|
| 56 |
+
*.npy
|
| 57 |
+
|
| 58 |
+
unitree_sdk2_python
|
| 59 |
+
|
| 60 |
+
dataset/
|
.gitmodules
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
[submodule "third_party/SIMPLE"]
|
| 2 |
+
path = third_party/SIMPLE
|
| 3 |
+
url = git@github.com:songlin/SIMPLE.git
|
LICENSE
ADDED
|
@@ -0,0 +1,17 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
Apache License
|
| 2 |
+
Version 2.0, January 2004
|
| 3 |
+
http://www.apache.org/licenses/
|
| 4 |
+
|
| 5 |
+
Copyright 2026 PSI-lab, USC
|
| 6 |
+
|
| 7 |
+
Licensed under the Apache License, Version 2.0 (the "License");
|
| 8 |
+
you may not use this file except in compliance with the License.
|
| 9 |
+
You may obtain a copy of the License at
|
| 10 |
+
|
| 11 |
+
http://www.apache.org/licenses/LICENSE-2.0
|
| 12 |
+
|
| 13 |
+
Unless required by applicable law or agreed to in writing, software
|
| 14 |
+
distributed under the License is distributed on an "AS IS" BASIS,
|
| 15 |
+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
| 16 |
+
See the License for the specific language governing permissions and
|
| 17 |
+
limitations under the License.
|
Psi0-训练阶段-state-action总览.md
ADDED
|
@@ -0,0 +1,216 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
---
|
| 2 |
+
tags:
|
| 3 |
+
- psi0
|
| 4 |
+
- training
|
| 5 |
+
- state-action
|
| 6 |
+
---
|
| 7 |
+
|
| 8 |
+
# Psi0 训练阶段 state/action 总览
|
| 9 |
+
|
| 10 |
+
> [!summary]
|
| 11 |
+
> 当前开源代码里,`pretrain`、`posttrain`、`finetune-real`、`finetune-simple` 都不是“显式全身关节输入 + 显式全身关节输出”。
|
| 12 |
+
> 更接近的实际接口是:
|
| 13 |
+
> - 上肢:手 + 手臂
|
| 14 |
+
> - 躯干:`rpy` + `height`
|
| 15 |
+
> - 移动相关:`torso_vx` / `torso_vy` / `torso_vyaw` / `target_yaw`
|
| 16 |
+
> - 下肢:更多像交给低层 controller,而不是由这层模型直接输出腿关节轨迹
|
| 17 |
+
|
| 18 |
+
## 1. 一页结论
|
| 19 |
+
|
| 20 |
+
| 阶段 | 模型输入 `state` | 模型输出 `action` | 是否显式包含腿关节输出 | 依据 |
|
| 21 |
+
| --- | --- | --- | --- | --- |
|
| 22 |
+
| `pretrain` `EgoDex` | `48` 维手部/腕部表示 | `48` 维手部/腕部表示 | 否 | `src/psi/data/egodex/egodex_dataset.py:308-330, 519-523` |
|
| 23 |
+
| `pretrain` `HE` | `hand_joints + arm_joints` | 左右腕位姿增量 + 双手指尖轨迹 | 否 | `src/psi/config/transform.py:112-212, 221-230` |
|
| 24 |
+
| `posttrain` `HE` | `hand_joints + arm_joints` | `action.joint_angles`,但当前接口仍是上肢维度 | 否 | `src/psi/data/humanoid/he_raw_dataset.py:206-229`;`src/psi/config/transform.py:718-755` |
|
| 25 |
+
| `finetune-real` | `hand + arm + torso_rpy + torso_height` | `hand + arm + rpy + height + vx + vy + vyaw + target_yaw` | 否 | `scripts/data/raw_to_lerobot.py:203-212, 239-258` |
|
| 26 |
+
| `finetune-simple` | `32` 维 psi-style `states`,本质是 `hand + arm + rpy + height` | `36` 维动作,核心仍是上肢 + 躯干/移动命令 | 否 | `src/gr00t/gr00t/deploy/gr00t_serve_simple.py:193-219, 133-166`;`src/psi/config/transform.py:376-393` |
|
| 27 |
+
|
| 28 |
+
## 2. 各阶段拆开看
|
| 29 |
+
|
| 30 |
+
### 2.1 `pretrain` on `EgoDex`
|
| 31 |
+
|
| 32 |
+
- `states` 是当前动作表示,不是全身关节。
|
| 33 |
+
- `actions` 是未来动作 chunk,也不是全身关节。
|
| 34 |
+
- 代码里直接写的是 `48d` action representation。
|
| 35 |
+
|
| 36 |
+
维度含义:
|
| 37 |
+
- 左腕
|
| 38 |
+
- 左手指尖
|
| 39 |
+
- 右腕
|
| 40 |
+
- 右手指尖
|
| 41 |
+
|
| 42 |
+
依据:
|
| 43 |
+
- `src/psi/data/egodex/egodex_dataset.py:308-330`
|
| 44 |
+
- `src/psi/data/egodex/egodex_dataset.py:519-523`
|
| 45 |
+
|
| 46 |
+
结论:
|
| 47 |
+
- 这是手部/腕部操作表示,不是全身关节表示。
|
| 48 |
+
|
| 49 |
+
### 2.2 `pretrain` on `HE`
|
| 50 |
+
|
| 51 |
+
- `state` 由 `observation.hand_joints` 和 `observation.arm_joints` 拼起来。
|
| 52 |
+
- `action` 由左右腕位姿和双手指尖轨迹拼起来。
|
| 53 |
+
- 没有把腿关节放进这条训练链路。
|
| 54 |
+
|
| 55 |
+
依据:
|
| 56 |
+
- `src/psi/config/transform.py:221-230`
|
| 57 |
+
- `src/psi/config/transform.py:112-212`
|
| 58 |
+
|
| 59 |
+
结论:
|
| 60 |
+
- 这条线仍然是上肢操作表示,不是全身关节表示。
|
| 61 |
+
|
| 62 |
+
### 2.3 `posttrain` on `HE`
|
| 63 |
+
|
| 64 |
+
- 原始 HE dataset loader 只读:
|
| 65 |
+
- `hand_state`
|
| 66 |
+
- `arm_state`
|
| 67 |
+
- 没有在这里读 `leg_state` 进入训练样本。
|
| 68 |
+
- `HEPosttrainRepackTransform` 里:
|
| 69 |
+
- `states = hand_joints + arm_joints`
|
| 70 |
+
- `actions = action.joint_angles`
|
| 71 |
+
- 但当前接口和脚本配置仍然是上肢维度,不是完整腿部关节轨迹接口。
|
| 72 |
+
|
| 73 |
+
依据:
|
| 74 |
+
- `src/psi/data/humanoid/he_raw_dataset.py:206-229`
|
| 75 |
+
- `src/psi/config/transform.py:718-755`
|
| 76 |
+
|
| 77 |
+
结论:
|
| 78 |
+
- `posttrain` 也不是显式全身关节训练。
|
| 79 |
+
|
| 80 |
+
### 2.4 `finetune-real`
|
| 81 |
+
|
| 82 |
+
- 数据转换脚本确实读了 `leg_state`,也把 `sol_q` 拆出了 `leg_joints`。
|
| 83 |
+
- 但最后真正写进 `states` / `action` 的内容里,没有把腿关节放进去。
|
| 84 |
+
|
| 85 |
+
`states` 实际组成:
|
| 86 |
+
- `hand_joints`
|
| 87 |
+
- `arm_joints`
|
| 88 |
+
- `torso_rpy`
|
| 89 |
+
- `torso_height`
|
| 90 |
+
|
| 91 |
+
`action` 实际组成:
|
| 92 |
+
- `hand_joints`
|
| 93 |
+
- `arm_joints`
|
| 94 |
+
- `rpy`
|
| 95 |
+
- `height`
|
| 96 |
+
- `torso_vx`
|
| 97 |
+
- `torso_vy`
|
| 98 |
+
- `torso_vyaw`
|
| 99 |
+
- `target_yaw`
|
| 100 |
+
|
| 101 |
+
依据:
|
| 102 |
+
- `scripts/data/raw_to_lerobot.py:203-212`
|
| 103 |
+
- `scripts/data/raw_to_lerobot.py:239-258`
|
| 104 |
+
|
| 105 |
+
训练器对 `36` 维动作的切分也和上面一致:
|
| 106 |
+
- `0:14` 手
|
| 107 |
+
- `14:28` 臂
|
| 108 |
+
- `28:31` `rpy`
|
| 109 |
+
- `31:32` `height`
|
| 110 |
+
- `32:33` `vx`
|
| 111 |
+
- `33:34` `vy`
|
| 112 |
+
- `34:35` `vyaw`
|
| 113 |
+
- `35:36` `target_yaw`
|
| 114 |
+
|
| 115 |
+
依据:
|
| 116 |
+
- `src/psi/trainers/finetune.py:492-520`
|
| 117 |
+
|
| 118 |
+
脚本配置也写死了 `36` 维:
|
| 119 |
+
- `scripts/train/psi0/finetune-real-psi0.sh:50-72`
|
| 120 |
+
|
| 121 |
+
补充:
|
| 122 |
+
- 可视化脚本也把 real 数据的腿关节直接置零,不从预测动作里取。
|
| 123 |
+
|
| 124 |
+
依据:
|
| 125 |
+
- `scripts/viz/viz_episode_real.py:73-75`
|
| 126 |
+
- `scripts/viz/viz_episode_real.py:90-94`
|
| 127 |
+
|
| 128 |
+
结论:
|
| 129 |
+
- `finetune-real` 不是“全身关节预测”。
|
| 130 |
+
- 更准确地说,是“上肢关节 + 躯干/移动命令预测”。
|
| 131 |
+
|
| 132 |
+
### 2.5 `finetune-simple`
|
| 133 |
+
|
| 134 |
+
- `simple` 原始观测里有腿关节,这点是有的。
|
| 135 |
+
- 但给 psi 模型用的 `states` 不是原始全身 `proprio`,而是压缩过的 psi-style 状态。
|
| 136 |
+
|
| 137 |
+
`states` 的 `32` 维组成:
|
| 138 |
+
- 左手 `7`
|
| 139 |
+
- 右手 `7`
|
| 140 |
+
- 左臂 `7`
|
| 141 |
+
- 右臂 `7`
|
| 142 |
+
- `rpy` `3`
|
| 143 |
+
- `height` `1`
|
| 144 |
+
|
| 145 |
+
依据:
|
| 146 |
+
- `src/gr00t/gr00t/deploy/gr00t_serve_simple.py:193-219`
|
| 147 |
+
|
| 148 |
+
`action` 在 SIMPLE 服务侧被拆成:
|
| 149 |
+
- `left_hand`
|
| 150 |
+
- `right_hand`
|
| 151 |
+
- `left_arm`
|
| 152 |
+
- `right_arm`
|
| 153 |
+
- `rpy`
|
| 154 |
+
- `height`
|
| 155 |
+
- `torso_vx`
|
| 156 |
+
- `torso_vy`
|
| 157 |
+
- `torso_vyaw`
|
| 158 |
+
- `target_yaw`
|
| 159 |
+
|
| 160 |
+
依据:
|
| 161 |
+
- `src/gr00t/gr00t/deploy/gr00t_serve_simple.py:133-166`
|
| 162 |
+
|
| 163 |
+
psi 的 `SimpleRepackTransform` 也只是把 `states` / `action` 取进来,再 pad 到训练���度:
|
| 164 |
+
- `src/psi/config/transform.py:376-393`
|
| 165 |
+
|
| 166 |
+
训练脚本同样固定成 `36` 维:
|
| 167 |
+
- `scripts/train/psi0/finetune-simple-psi0.sh:50-72`
|
| 168 |
+
|
| 169 |
+
可视化脚本里:
|
| 170 |
+
- `proprio` 会显示腿关节
|
| 171 |
+
- 但 `action` 里的腿是直接用当前观测腿关节,而不是用模型预测结果
|
| 172 |
+
|
| 173 |
+
依据:
|
| 174 |
+
- `scripts/viz/viz_episode_simple.py:83-85`
|
| 175 |
+
- `scripts/viz/viz_episode_simple.py:93-96`
|
| 176 |
+
|
| 177 |
+
结论:
|
| 178 |
+
- `finetune-simple` 这条线也不是“模型直接输出腿关节轨迹”。
|
| 179 |
+
|
| 180 |
+
## 3. 下肢到底在哪里
|
| 181 |
+
|
| 182 |
+
> [!note]
|
| 183 |
+
> README 的说法和代码里的实际接口,要分开理解。
|
| 184 |
+
|
| 185 |
+
README 里写的是:
|
| 186 |
+
- `System-1` 预测 future whole-body action chunks
|
| 187 |
+
- `System-0` 执行 predicted lower-body action commands
|
| 188 |
+
|
| 189 |
+
依据:
|
| 190 |
+
- `README.md:26`
|
| 191 |
+
|
| 192 |
+
结合实际代码,更合理的理解是:
|
| 193 |
+
- `System-1` 负责高层动作生成
|
| 194 |
+
- 当前开源训练接口里,显式暴露给模型训练的主要是上肢和躯干/移动命令
|
| 195 |
+
- 下肢稳定控制更多由 `System-0` 的 tracking controller 负责
|
| 196 |
+
|
| 197 |
+
所以如果问题是:
|
| 198 |
+
|
| 199 |
+
**“这份代码里有没有直接训练/输出全身每个关节?”**
|
| 200 |
+
|
| 201 |
+
答案是:
|
| 202 |
+
- 没看到。
|
| 203 |
+
|
| 204 |
+
**“有没有和移动、下肢相关的信息?”**
|
| 205 |
+
|
| 206 |
+
答案是:
|
| 207 |
+
- 有,但更像高层 locomotion command,不是显式腿关节序列。
|
| 208 |
+
|
| 209 |
+
## 4. 最短版记忆卡
|
| 210 |
+
|
| 211 |
+
- `pretrain(EgoDex)`:手/腕 `48D`
|
| 212 |
+
- `pretrain(HE)`:`hand + arm`,动作是腕位姿 + 指尖轨迹
|
| 213 |
+
- `posttrain(HE)`:`hand + arm`
|
| 214 |
+
- `finetune-real`:`hand + arm + rpy + height + vx/vy/vyaw/target_yaw`
|
| 215 |
+
- `finetune-simple`:原始观测有腿,但 psi 模型接口仍不是显式腿关节输出
|
| 216 |
+
- 结论:当前开源接口不是“全身关节端到端直接预测”
|
README.md
ADDED
|
@@ -0,0 +1,532 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
<h1 align="center">Ψ₀: An Open Foundation Model <br/> Towards Universal Humanoid Loco-Manipulation
|
| 2 |
+
</h1>
|
| 3 |
+
|
| 4 |
+
<p align="center">
|
| 5 |
+
<img src="assets/media/teaser.jpg" alt="Psi0 teaser image" />
|
| 6 |
+
</p>
|
| 7 |
+
|
| 8 |
+
<div align="center">
|
| 9 |
+
|
| 10 |
+
[](https://arxiv.org/abs/2603.12263)
|
| 11 |
+
[](https://psi-lab.ai/Psi0)
|
| 12 |
+
[](https://huggingface.co/USC-PSI-Lab/psi-model)
|
| 13 |
+
[](https://huggingface.co/datasets/USC-PSI-Lab/psi-data)
|
| 14 |
+
[](./LICENSE)
|
| 15 |
+
|
| 16 |
+
</div>
|
| 17 |
+
|
| 18 |
+
Contributors: [Songlin Wei](https://songlin.github.io/), [Hongyi Jing](https://hongyijing.me/), [Boqian Li](https://boqian-li.github.io/), [Zhenyu Zhao](https://zhenyuzhao.com/), [Jiageng Mao](https://pointscoder.github.io/), [Zhenhao Ni](https://nizhenhao-3.github.io/) , [Sicheng He](https://hesicheng.net/), [Jie Liu](https://jie0530.github.io/), [Xiawei Liu](https://www.xiaweiliu.com/), Kaidi Kang, Sheng Zang,[Weiduo Yuan](https://weiduoyuan.com/), [Marco Pavone](https://profiles.stanford.edu/marco-pavone), Di Huang, [Yue Wang](https://yuewang.xyz/)
|
| 19 |
+
|
| 20 |
+
-------
|
| 21 |
+
|
| 22 |
+
$\Psi_0$ is an open vision-language-action (VLA) model for dexterous humanoid loco-manipulation. Our model first learns task semantics and visual representation from large-scale human egocentic videos, and then is post-trained on a smaller amount of real-world teleoperated robot data, to learn general dynamics of the embodiment.
|
| 23 |
+
|
| 24 |
+
Our foundation model is capable of acquiring new long-horizontal dexterous loco-manipulation skill by fine-tuning using as few as 80 trajectories. ***Our key finding is that scaling the right data in the right way.***
|
| 25 |
+
|
| 26 |
+
At the top, the $\Psi_0$ model consists of two end-to-end trained components: a vision–language backbone (System-2) and a multimodal diffusion transformer (System-1) action expert. The backbone is based on Qwen’s Qwen3-VL-2B-Instruct, which extracts vision–language features from observations and instructions. These features condition a flow-based multimodal diffusion transformer inspired by Stable Diffusion 3. The action expert (≈500M parameters) predicts future whole-body action chunks, enabling efficient fusion of visual, linguistic, and action representations. At the lowest level (System-0), an RL-based tracking controller executes the predicted lower-body action commands, ensuring stable and precise physical control.
|
| 27 |
+
|
| 28 |
+
<p align="center">
|
| 29 |
+
<img src="assets/media/arch.png" alt="Psi0 model" />
|
| 30 |
+
</p>
|
| 31 |
+
|
| 32 |
+
## Table of Contents
|
| 33 |
+
<!-- - [Installation](#-environment-setup) -->
|
| 34 |
+
<!-- - [Pre- & Post- Training](#-) -->
|
| 35 |
+
<!-- - [Data Pre-Processing](#-) -->
|
| 36 |
+
- [Finetune Ψ₀ on Unitree G1 Humanoid Robot](#finetune-psi0)
|
| 37 |
+
- [Installation](#installation)
|
| 38 |
+
- [Data Collection](#data-collection)
|
| 39 |
+
- [Fine-Tuning](#training-real)
|
| 40 |
+
- [Open-Loop Evaluation](#open-loop-evaluation)
|
| 41 |
+
- [Deployment](#deployment)
|
| 42 |
+
- [Baselines](#baselines)
|
| 43 |
+
- [GR00T N1.6](#groot-n16)
|
| 44 |
+
- [OpenPi π0.5](#openpi-05)
|
| 45 |
+
- [InternVLA-M1](#internvla-m1)
|
| 46 |
+
- [H-RDT](#-)
|
| 47 |
+
- [EgoVLA](#-)
|
| 48 |
+
- [Diffusion Policy](#-)
|
| 49 |
+
- [ACT](#-)
|
| 50 |
+
- [Simulation](#simulation)
|
| 51 |
+
- [Install SIMPLE](#install-simple)
|
| 52 |
+
- [Data Generation](#data-generation)
|
| 53 |
+
- [Fine-Tuning](#training-sim)
|
| 54 |
+
- [Evaluation in SIMPLE](#evaluation-in-simple)
|
| 55 |
+
- [Reproduce Ψ₀: Pre-Training and Post-Training](#pre-post-train)
|
| 56 |
+
- [Checkpoints](#checkpoints)
|
| 57 |
+
- [Troubleshootings](#troubleshootings)
|
| 58 |
+
- [Citation](#️-citation)
|
| 59 |
+
|
| 60 |
+
<a id="finetune-psi0"></a>
|
| 61 |
+
## Finetune Ψ₀ on Unitree G1 Humanoid Robot
|
| 62 |
+
|
| 63 |
+
### Installation
|
| 64 |
+
|
| 65 |
+
Clone the project and change directory to the project root:
|
| 66 |
+
```bash
|
| 67 |
+
git clone git@github.com:physical-superintelligence-lab/Psi0.git
|
| 68 |
+
cd Psi0
|
| 69 |
+
```
|
| 70 |
+
We use [uv](https://docs.astral.sh/uv/getting-started/installation/) to manage Python dependencies. Install `uv` if not already installed:
|
| 71 |
+
|
| 72 |
+
```bash
|
| 73 |
+
curl -LsSf https://astral.sh/uv/install.sh | sh
|
| 74 |
+
```
|
| 75 |
+
|
| 76 |
+
Set up the $\Psi_0$ environment:
|
| 77 |
+
|
| 78 |
+
> ℹ️ We manage the $\Psi_0$ environment and all the baselines through `uv` and they all share the same `src/` code. See [Environment Management](baselines/README.md) for more details.
|
| 79 |
+
|
| 80 |
+
```
|
| 81 |
+
uv venv .venv-psi --python 3.10
|
| 82 |
+
source .venv-psi/bin/activate
|
| 83 |
+
GIT_LFS_SKIP_SMUDGE=1 uv sync --all-groups --index-strategy unsafe-best-match --active
|
| 84 |
+
uv pip install flash_attn==2.7.4.post1 --no-build-isolation
|
| 85 |
+
```
|
| 86 |
+
|
| 87 |
+
Test installation, a version number should be displayed.
|
| 88 |
+
```bash
|
| 89 |
+
python -c "import psi;print(psi.__version__)"
|
| 90 |
+
```
|
| 91 |
+
|
| 92 |
+
Verify the shared `lerobot` stack is importable.
|
| 93 |
+
```bash
|
| 94 |
+
python -c "from psi.data.lerobot.compat import LEROBOT_LAYOUT; print(LEROBOT_LAYOUT)"
|
| 95 |
+
```
|
| 96 |
+
|
| 97 |
+
### Data Collection
|
| 98 |
+
> 📂 We open-sourced all the 9 real-world tasks. You can directly download the data and jump to the [Fine-Tuning](#training-real).
|
| 99 |
+
|
| 100 |
+
> 🔥 We first release our internal test data collection pipeline which uses Apple Vision Pro to teleoperate Unitree G1 humanoid Robot with two Dex3-1 hands.
|
| 101 |
+
|
| 102 |
+
See the detailed teleoperation guide here:
|
| 103 |
+
[Real-World Teleoperation Guide](real/README.md)
|
| 104 |
+
|
| 105 |
+
|
| 106 |
+
#### Pre-Processing: Convert Raw Data to LeRobot Format
|
| 107 |
+
|
| 108 |
+
```
|
| 109 |
+
export task=Hug_box_and_move
|
| 110 |
+
|
| 111 |
+
hf download USC-PSI-Lab/psi-data \
|
| 112 |
+
g1_real_raw/$task.zip \
|
| 113 |
+
--local-dir=$PSI_HOME/data/real_teleop_g1 \
|
| 114 |
+
--repo-type=dataset
|
| 115 |
+
|
| 116 |
+
unzip $PSI_HOME/data/real_teleop_g1/g1_real_raw/$task.zip -d $PSI_HOME/data/real_teleop_g1/g1_real_raw/$task
|
| 117 |
+
```
|
| 118 |
+
You should observe similar folder structure:
|
| 119 |
+
|
| 120 |
+
```
|
| 121 |
+
g1_real_raw
|
| 122 |
+
└── Hug_box_and_move
|
| 123 |
+
├── episode_0
|
| 124 |
+
│ ├── color
|
| 125 |
+
│ │ ├── frame_000000.jpg
|
| 126 |
+
│ │ └── ...
|
| 127 |
+
│ └── data.json
|
| 128 |
+
└── ...
|
| 129 |
+
```
|
| 130 |
+
|
| 131 |
+
Edit the task description file with the following format, eg.,
|
| 132 |
+
```
|
| 133 |
+
vim scripts/data/task_description_dict.json
|
| 134 |
+
```
|
| 135 |
+
```
|
| 136 |
+
{
|
| 137 |
+
"Hug_box_and_move": "Hug box and move."
|
| 138 |
+
}
|
| 139 |
+
```
|
| 140 |
+
|
| 141 |
+
Run conversion script
|
| 142 |
+
```
|
| 143 |
+
python scripts/data/raw_to_lerobot.py \
|
| 144 |
+
--data-root=/hfm/data/real_teleop_g1/g1_real_raw \
|
| 145 |
+
--work-dir=/hfm/data/real \
|
| 146 |
+
--repo-id=psi0-real-g1 \
|
| 147 |
+
--robot-type=g1 \
|
| 148 |
+
--task=$task
|
| 149 |
+
```
|
| 150 |
+
|
| 151 |
+
Calculate stats
|
| 152 |
+
```
|
| 153 |
+
python scripts/data/calc_modality_stats.py \
|
| 154 |
+
--work-dir=$PSI_HOME/data/real \
|
| 155 |
+
--task=$task
|
| 156 |
+
```
|
| 157 |
+
|
| 158 |
+
Create **$\Psi_0$** format stats (simply a copy for now)
|
| 159 |
+
```
|
| 160 |
+
cp $PSI_HOME/data/real/$task/meta/stats.json $PSI_HOME/data/real/$task/meta/stats_psi0.json
|
| 161 |
+
```
|
| 162 |
+
|
| 163 |
+
Now it's ready to finetune $\Psi_0$.
|
| 164 |
+
|
| 165 |
+
> ✈️ If training env is already configured, directly launch training via `scripts/train/psi0/finetune-real-psi0.sh $task`
|
| 166 |
+
|
| 167 |
+
|
| 168 |
+
<a id="training-real"></a>
|
| 169 |
+
### Fine-Tuning
|
| 170 |
+
|
| 171 |
+
> ✔️ Suppose the data is already collected and processed. Now we can proceed to fine-tune the $\Psi_0$ model.
|
| 172 |
+
|
| 173 |
+
> There is a [known issue](https://github.com/physical-superintelligence-lab/Psi0/issues/3) of loading our real data, apply this fix first `python scripts/data/patch_lerobot_meta.py $PSI_HOME/data/real/$task`
|
| 174 |
+
|
| 175 |
+
> 📝 Here we illustrate by using the pre-collected data from [Huggingface psi-data](https://huggingface.co/datasets/USC-PSI-Lab/psi-data/tree/main/real).
|
| 176 |
+
|
| 177 |
+
Set up the environment variables following `.env.sample`. The environment variables will be loaded by the `dotenv.load_dotenv()` in python.
|
| 178 |
+
```
|
| 179 |
+
cp .env.sample .env
|
| 180 |
+
# and edit the following env variables
|
| 181 |
+
# HF_TOKEN=<YOUR HF READ TOKEN>
|
| 182 |
+
# WANDB_API_KEY=<API KEY for wandb logging>
|
| 183 |
+
# WANDB_ENTITY=<wandb entity>
|
| 184 |
+
# PSI_HOME=<Path where PSI cache/checkpoint/data are located by convention>
|
| 185 |
+
|
| 186 |
+
source .env
|
| 187 |
+
echo $PSI_HOME
|
| 188 |
+
```
|
| 189 |
+
|
| 190 |
+
Download the collected real-world data and extract it:
|
| 191 |
+
```
|
| 192 |
+
export task=Pick_bottle_and_turn_and_pour_into_cup
|
| 193 |
+
|
| 194 |
+
hf download USC-PSI-Lab/psi-data \
|
| 195 |
+
real/$task.zip \
|
| 196 |
+
--local-dir=$PSI_HOME/data \
|
| 197 |
+
--repo-type=dataset
|
| 198 |
+
|
| 199 |
+
unzip $PSI_HOME/data/real/$task.zip -d $PSI_HOME/data/real
|
| 200 |
+
```
|
| 201 |
+
> 👀 If you want to visualize the episode please refer to the [Data Visualization](examples/visualize.md) in the examples.
|
| 202 |
+
|
| 203 |
+
Launch the training script:
|
| 204 |
+
```
|
| 205 |
+
scripts/train/psi0/finetune-real-psi0.sh $task
|
| 206 |
+
```
|
| 207 |
+
|
| 208 |
+
> 🖥️ You can always change the GPUs, e.g., `CUDA_VISIBLE_DEVICES=0,1,2,3 scripts/train/...`.
|
| 209 |
+
|
| 210 |
+
> ⚠️ Please try to maintain a reasonable global batch size = device batch size x number of GPUs x gradient accumulation step. We use global batch size 128 throughout all the real-world and simulation experiments.
|
| 211 |
+
|
| 212 |
+
|
| 213 |
+
### Open-Loop Evaluation
|
| 214 |
+
> Follow the steps in `examples/simple/openloop_eval.ipynb`
|
| 215 |
+
|
| 216 |
+
Load the training dataset, and run model inference to see how model fits the training data.
|
| 217 |
+
|
| 218 |
+
### Deployment
|
| 219 |
+
|
| 220 |
+
#### Serve $\Psi_0$ (RTC mode)
|
| 221 |
+
|
| 222 |
+
```bash
|
| 223 |
+
bash ./scripts/deploy/serve_psi0-rtc.sh
|
| 224 |
+
```
|
| 225 |
+
|
| 226 |
+
#### Start $\Psi_0$ Client (RTC mode)
|
| 227 |
+
|
| 228 |
+
```bash
|
| 229 |
+
bash ./real/scripts/deploy_psi0-rtc.sh
|
| 230 |
+
```
|
| 231 |
+
|
| 232 |
+
For detailed real-world deployment environment setup, please also refer to the dedicated documentation:
|
| 233 |
+
|
| 234 |
+
[Real-World Teleoperation Guide](real/README.md)
|
| 235 |
+
|
| 236 |
+
|
| 237 |
+
## Baselines
|
| 238 |
+
|
| 239 |
+
<a id="groot-n16"></a>
|
| 240 |
+
|
| 241 |
+
### GR00T
|
| 242 |
+
Install the env
|
| 243 |
+
```bash
|
| 244 |
+
cd src/gr00t; uv sync
|
| 245 |
+
```
|
| 246 |
+
1. training
|
| 247 |
+
```bash
|
| 248 |
+
cd src/gr00t
|
| 249 |
+
./scripts/train_gr00t.sh --dataset-path /your/lerobot/dataset
|
| 250 |
+
```
|
| 251 |
+
2. serving a checkpoint
|
| 252 |
+
```bash
|
| 253 |
+
cd src/gr00t
|
| 254 |
+
./scripts/deploy_gr00t.sh
|
| 255 |
+
```
|
| 256 |
+
|
| 257 |
+
3. openloop eval on trained checkpoint using gt
|
| 258 |
+
```bash
|
| 259 |
+
cd src/gr00t
|
| 260 |
+
./scripts/openloop_eval.sh
|
| 261 |
+
```
|
| 262 |
+
|
| 263 |
+
<a id="openpi-05"></a>
|
| 264 |
+
|
| 265 |
+
### OpenPI $\pi_{0.5}$
|
| 266 |
+
|
| 267 |
+
Please see more detailed instructions here: [baselines/pi05](baselines/pi05/README.md).
|
| 268 |
+
|
| 269 |
+
### InternVLA-M1
|
| 270 |
+
Install the env
|
| 271 |
+
```bash
|
| 272 |
+
cd src/InternVLA-M1; uv sync --python 3.10
|
| 273 |
+
```
|
| 274 |
+
1. training
|
| 275 |
+
```bash
|
| 276 |
+
cd src/InternVLA-M1
|
| 277 |
+
bash scripts/train_internvla.sh
|
| 278 |
+
```
|
| 279 |
+
2. serving a checkpoint
|
| 280 |
+
```bash
|
| 281 |
+
cd src/InternVLA-M1
|
| 282 |
+
./scripts/deploy_internvla.sh
|
| 283 |
+
```
|
| 284 |
+
|
| 285 |
+
## Simulation
|
| 286 |
+
|
| 287 |
+
We use [SIMPLE]() to benchmark $\Psi_0$ and all the baselines.
|
| 288 |
+
|
| 289 |
+
> 📢 SIMPLE is an easy-to-use humanoid benchmarking simulator built on the MuJoCo physics engine and Isaac Sim rendering.
|
| 290 |
+
|
| 291 |
+
### Install SIMPLE
|
| 292 |
+
[Coming soon]
|
| 293 |
+
|
| 294 |
+
### Data Generation
|
| 295 |
+
> 📂 We also provide 5 pre-collected whole-body humanoid loco-manipulation tasks at [Huggingface psi-data](https://huggingface.co/datasets/USC-PSI-Lab/psi-data/tree/main/real). If you want to use the existing simulation data, jump to the [Fine-Tuning](#training-sim)
|
| 296 |
+
|
| 297 |
+
#### Motion-Planning Based Data Generation
|
| 298 |
+
[Coming soon]
|
| 299 |
+
|
| 300 |
+
#### Teleoperation in Simulator
|
| 301 |
+
[Coming soon]
|
| 302 |
+
|
| 303 |
+
<a id="training-sim"></a>
|
| 304 |
+
### Fine-Tuning
|
| 305 |
+
|
| 306 |
+
Download [SIMPLE task data](https://huggingface.co/datasets/USC-PSI-Lab/psi-data/tree/main/simple) and extract it:
|
| 307 |
+
|
| 308 |
+
> 💡 Dont forget `source .env` first before following below commands.
|
| 309 |
+
|
| 310 |
+
```
|
| 311 |
+
export task=G1WholebodyBendPick-v0
|
| 312 |
+
|
| 313 |
+
hf download USC-PSI-Lab/psi-data \
|
| 314 |
+
simple/$task$.zip \
|
| 315 |
+
--local-dir=$PSI_HOME/data \
|
| 316 |
+
--repo-type=dataset
|
| 317 |
+
|
| 318 |
+
unzip $PSI_HOME/data/simple/$task.zip -d $PSI_HOME/data/simple
|
| 319 |
+
```
|
| 320 |
+
|
| 321 |
+
> 👀 If you want to visualize the episode please refer to the [Data Visualization](examples/visualize.md) in the examples.
|
| 322 |
+
|
| 323 |
+
Start training:
|
| 324 |
+
|
| 325 |
+
> Please [set up the envrionment variables](#training-real) if not done so yet.
|
| 326 |
+
|
| 327 |
+
```
|
| 328 |
+
bash scripts/train/psi0/finetune-simple-psi0.sh $task
|
| 329 |
+
```
|
| 330 |
+
The training will create a run dir which is located under `.runs` in the project root.
|
| 331 |
+
If your GPU has limited VRAM, set `--train.optimizer-foreach=false` to reduce optimizer-step memory usage at the cost of some speed.
|
| 332 |
+
|
| 333 |
+
### Evaluation in SIMPLE
|
| 334 |
+
|
| 335 |
+
#### Serve $\Psi_0$
|
| 336 |
+
```
|
| 337 |
+
export run_dir=<the run dir here under folder .runs>
|
| 338 |
+
export ckpt_step=<checkpoint step>
|
| 339 |
+
uv run --active --group psi --group serve serve_psi0 \
|
| 340 |
+
--host 0.0.0.0 \
|
| 341 |
+
--port 22085 \
|
| 342 |
+
--run-dir=$run_dir \
|
| 343 |
+
--ckpt-step=$ckpt_step \
|
| 344 |
+
--action-exec-horizon=24 \
|
| 345 |
+
--rtc
|
| 346 |
+
```
|
| 347 |
+
|
| 348 |
+
|
| 349 |
+
Run open-loop evaluation (offline)
|
| 350 |
+
|
| 351 |
+
[examples/simple/openloop_eval.ipynb](examples/simple/openloop_eval.ipynb)
|
| 352 |
+
|
| 353 |
+
#### Run the Evaluation Client
|
| 354 |
+
> If the server is started on a remote server, run ssh port forward. eg., `ssh -L 22086:localhost:22086 songlin@nebula100`.
|
| 355 |
+
|
| 356 |
+
> Once port forward is done, open a new terminal to test if server is up `curl -i http://localhost:22085/health`
|
| 357 |
+
|
| 358 |
+
Launch the eval client through docker
|
| 359 |
+
```
|
| 360 |
+
GPUs=1 docker compose run eval $task psi0 \
|
| 361 |
+
--host=localhost \
|
| 362 |
+
--port=22085 \
|
| 363 |
+
--sim-mode=mujoco_isaac \
|
| 364 |
+
--headless \
|
| 365 |
+
--max-episode-steps=360 \
|
| 366 |
+
--num-episodes=10 \
|
| 367 |
+
--data-format=lerobot \
|
| 368 |
+
--data-dir=data/$task
|
| 369 |
+
```
|
| 370 |
+
The policy rollout videos will be found in folder `third_party/SIMPLE/data/evals/psi0`.
|
| 371 |
+
|
| 372 |
+
> The evaluation for a single episode could take up to 6~10 minutes because SIMPLE use a synchronous rendering API in IsaacSim. See here for [more explanation](#).
|
| 373 |
+
|
| 374 |
+
<a id="pre-post-train"></a>
|
| 375 |
+
## Reproduce Ψ₀: Pre-Training and Post-Training
|
| 376 |
+
|
| 377 |
+
|
| 378 |
+
### Pre-Train VLM
|
| 379 |
+
|
| 380 |
+
Download and cache the official `Qwen/Qwen3-VL-2B-Instruct` weights.
|
| 381 |
+
```
|
| 382 |
+
scripts/predownload_qwen3vl.py
|
| 383 |
+
```
|
| 384 |
+
|
| 385 |
+
Pre-train on the [EgoDex dataset](https://github.com/apple/ml-egodex)
|
| 386 |
+
|
| 387 |
+
Pre-compute `48 DoF EgoDex action`:
|
| 388 |
+
|
| 389 |
+
> We re-use the pre-process code from [H-RDT EgoDex Pre-Processing](https://github.com/HongzheBi/H_RDT?tab=readme-ov-file#data-preprocessing).
|
| 390 |
+
> 1. Change the paths in `src/h_rdt/datasets/pretrain/setup_pretrain.sh`.
|
| 391 |
+
> 2. Tweak the `NUM_PROCESSES` if on a powerful server, i tried max 64.
|
| 392 |
+
> 3. set `FORCE_OVERWRITE=True` if the processing script is disrupted.
|
| 393 |
+
|
| 394 |
+
```
|
| 395 |
+
source src/h_rdt/datasets/pretrain/setup_pretrain.sh
|
| 396 |
+
source .venv-psi/bin/activate
|
| 397 |
+
bash src/h_rdt/datasets/pretrain/run_pretrain_pipeline.sh
|
| 398 |
+
```
|
| 399 |
+
|
| 400 |
+
```
|
| 401 |
+
bash scripts/train/psi0/pretrain-egodex-psi0-fast.sh
|
| 402 |
+
```
|
| 403 |
+
|
| 404 |
+
Pre-train on [humanoid everyday dataset](https://huggingface.co/datasets/USC-GVL/humanoid-everyday)
|
| 405 |
+
|
| 406 |
+
> Please download the pre-processed HE data here: `hf download USC-PSI-Lab/psi-data HE_RAW.zip --repo-type=dataset`
|
| 407 |
+
|
| 408 |
+
```
|
| 409 |
+
bash scripts/train/psi0/pretrain-he-psi0-fast.sh
|
| 410 |
+
```
|
| 411 |
+
|
| 412 |
+
Save the pretrained checkpoints once training is done:
|
| 413 |
+
```
|
| 414 |
+
python scripts/save_pretrain_qwen3vl_backbone.py
|
| 415 |
+
```
|
| 416 |
+
|
| 417 |
+
### Post-Train Action Expert
|
| 418 |
+
|
| 419 |
+
Download pre-trained `psi-0` VLM backbone
|
| 420 |
+
```
|
| 421 |
+
python scripts/data/download.py \
|
| 422 |
+
--repo-id=USC-PSI-Lab/psi-model \
|
| 423 |
+
--remote-dir=pre.fast.egodex.2512241941/pretrained/ckpt_200000 \
|
| 424 |
+
--local-dir=/hfm/cache/checkpoints/psi0/pre.fast.egodex.2512241941.ckpt200k \
|
| 425 |
+
--repo-type=model
|
| 426 |
+
```
|
| 427 |
+
|
| 428 |
+
Post-train on [humanoid everyday (HE) dataset](https://huggingface.co/datasets/USC-GVL/humanoid-everyday)
|
| 429 |
+
```
|
| 430 |
+
bash scripts/train/psi0/posttrain-he-psi0.sh
|
| 431 |
+
```
|
| 432 |
+
|
| 433 |
+
Save post-trained action header once training is over
|
| 434 |
+
```
|
| 435 |
+
python scripts/save_posttrain_action_expert.py
|
| 436 |
+
```
|
| 437 |
+
|
| 438 |
+
## Checkpoints
|
| 439 |
+
|
| 440 |
+
The released checkpoints on [HuggingFace Psi-Model](https://huggingface.co/USC-PSI-Lab/psi-model) is listed
|
| 441 |
+
|
| 442 |
+
| Checkpoint | Description | Remote Directory |
|
| 443 |
+
|---|---|---|
|
| 444 |
+
| $\Psi_0$ VLM<br/>(Baseline) | Pre-trained VLM backbone (EgoDex 200K steps + HE 30K steps) | `psi0/pre.fast.1by1.2601091803.ckpt.ego200k.he30k` |
|
| 445 |
+
| $\Psi_0$ Action Expert<br/>(Baseline) | Post-trained Action Expert On HE | `psi0/postpre.1by1.pad36.2601131206.ckpt.he30k` |
|
| 446 |
+
|
| 447 |
+
and more variants for ablation studies:
|
| 448 |
+
| Checkpoint | Description | Remote Directory |
|
| 449 |
+
|---|---|---|
|
| 450 |
+
| $\Psi_0$ VLM<br/>(Ablation Study) | Pre-trained VLM backbone only on EgoDex 200K steps | `psi0/pre.fast.egodex.2512241941.ckpt200k` |
|
| 451 |
+
| $\Psi_0$ VLM<br/>(Ablation Study) | Pre-trained VLM backbone only on HE 48K steps | `psi0/pre.abl.only.he.2512311516.48k` |
|
| 452 |
+
| $\Psi_0$ VLM<br/>(Ablation Study) | Pre-trained VLM backbone only on 10% EgoDex | `psi0/pre.abl.ego.10per.2602021632.46k` |
|
| 453 |
+
| $\Psi_0$ Action Expert<br/>(Ablation Study) | Post-train on HE by picking pre-trained variant `psi0/pre.abl.only.he.2512311516.48k` | `psi0/postpre.abl.only.he.2602050012` |
|
| 454 |
+
| $\Psi_0$ Action Expert<br/>(Ablation Study) | Post-train on HE by picking pre-trained variant `psi0/pre.abl.ego.10per.2602021632.46k` | `psi0/postpre.abl.ego.10per.2602050006` |
|
| 455 |
+
|
| 456 |
+
|
| 457 |
+
Download the selected models
|
| 458 |
+
|
| 459 |
+
```
|
| 460 |
+
hf download USC-PSI-Lab/psi-model \
|
| 461 |
+
--remote-dir=<remote dictory on huggingface repo>
|
| 462 |
+
--local-dir=$PSI_HOME/cache/checkpoints \
|
| 463 |
+
--repo-type=model
|
| 464 |
+
```
|
| 465 |
+
|
| 466 |
+
## Troubleshootings
|
| 467 |
+
|
| 468 |
+
1. Lerobot dataset issues: `stack(): argument 'tensors' (position 1) must be tuple of Tensors, not Column`
|
| 469 |
+
|
| 470 |
+
This usually means the environment is still on the legacy PSI `lerobot` stack. Resync the PSI env so it uses the
|
| 471 |
+
same `lerobot` and `datasets` versions as SIMPLE, then verify the import layout:
|
| 472 |
+
|
| 473 |
+
```bash
|
| 474 |
+
source .venv-psi/bin/activate
|
| 475 |
+
uv sync --group psi --active
|
| 476 |
+
python -c "from psi.data.lerobot.compat import LEROBOT_LAYOUT; print(LEROBOT_LAYOUT)"
|
| 477 |
+
```
|
| 478 |
+
|
| 479 |
+
2. Fail to install `evdev`, `src/evdev/input.c:10:10: fatal error: Python.h: No such file or directory`
|
| 480 |
+
|
| 481 |
+
```
|
| 482 |
+
sudo apt update
|
| 483 |
+
sudo apt install -y python3-dev python3-venv build-essential \
|
| 484 |
+
linux-headers-$(uname -r)
|
| 485 |
+
```
|
| 486 |
+
|
| 487 |
+
3. RuntimeError: Could not load libtorchcodec. Likely causes ...
|
| 488 |
+
```
|
| 489 |
+
sudo apt-get install ffmpeg
|
| 490 |
+
```
|
| 491 |
+
|
| 492 |
+
4. ImportError: cannot import name 'Deprecated' from 'wandb.proto.wandb_telemetry_pb2'
|
| 493 |
+
|
| 494 |
+
re-install `wandb`
|
| 495 |
+
```
|
| 496 |
+
source .venv-pusht/bin/activate
|
| 497 |
+
uv pip uninstall wandb
|
| 498 |
+
uv pip install wandb==0.18.0
|
| 499 |
+
```
|
| 500 |
+
|
| 501 |
+
5. support `sm_120` on newer GPUs like `5090` or `RTX 6000`, UserWarning: Ignoring invalid value for boolean flag CUDA_LAUNCH_BLOCKING: truevalid values are 0 or 1.
|
| 502 |
+
|
| 503 |
+
update `torch` and `flash-attn`
|
| 504 |
+
```
|
| 505 |
+
uv pip install torch==2.7.0 torchvision==0.22.0 torchaudio==2.7.0 --index-url https://download.pytorch.org/whl/cu128
|
| 506 |
+
uv pip install flash-attn --no-build-isolation
|
| 507 |
+
```
|
| 508 |
+
|
| 509 |
+
6. Failed to download and build `lerobot ... `, Use `git lfs logs last` to view the log.
|
| 510 |
+
|
| 511 |
+
```
|
| 512 |
+
GIT_LFS_SKIP_SMUDGE=1 uv ...
|
| 513 |
+
```
|
| 514 |
+
## Citation
|
| 515 |
+
|
| 516 |
+
```
|
| 517 |
+
@misc{wei2026psi0,
|
| 518 |
+
title={$\Psi_0$: An Open Foundation Model Towards Universal Humanoid Loco-Manipulation},
|
| 519 |
+
author={Songlin Wei and Hongyi Jing and Boqian Li and Zhenyu Zhao and Jiageng Mao and Zhenhao Ni and Sicheng He and Jie Liu and Xiawei Liu and Kaidi Kang and Sheng Zang and Weiduo Yuan and Marco Pavone and Di Huang and Yue Wang},
|
| 520 |
+
year={2026},
|
| 521 |
+
eprint={2603.12263},
|
| 522 |
+
archivePrefix={arXiv},
|
| 523 |
+
primaryClass={cs.RO},
|
| 524 |
+
url={https://arxiv.org/abs/2603.12263},
|
| 525 |
+
}
|
| 526 |
+
```
|
| 527 |
+
|
| 528 |
+
## License
|
| 529 |
+
|
| 530 |
+
This project is licensed under the Apache License 2.0.
|
| 531 |
+
|
| 532 |
+
See the [LICENSE](LICENSE) file for details.
|
cache_dance_video_omni_embeds.sh
ADDED
|
@@ -0,0 +1,13 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#!/usr/bin/env bash
|
| 2 |
+
set -euo pipefail
|
| 3 |
+
|
| 4 |
+
cd /cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/Psi0-yth-v13
|
| 5 |
+
|
| 6 |
+
export BUILD_DANCE=1
|
| 7 |
+
export BUILD_MOTION_EXAMPLE=0
|
| 8 |
+
export CACHE_DANCE=1
|
| 9 |
+
export CACHE_MOTION_EXAMPLE=0
|
| 10 |
+
export OVERWRITE_CLIPS=${OVERWRITE_CLIPS:-1}
|
| 11 |
+
export OVERWRITE_EMBEDS=${OVERWRITE_EMBEDS:-1}
|
| 12 |
+
|
| 13 |
+
exec bash cache_video_omni_embeds.sh
|
cache_motion_example_video_omni_embeds.sh
ADDED
|
@@ -0,0 +1,22 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#!/usr/bin/env bash
|
| 2 |
+
set -euo pipefail
|
| 3 |
+
|
| 4 |
+
cd /cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/Psi0-yth-v18
|
| 5 |
+
|
| 6 |
+
export PSI_VIDEO_CLIP_DATASET_ROOT=${PSI_VIDEO_CLIP_DATASET_ROOT:-/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/videogen/psi0_video_clip_50f_rand8}
|
| 7 |
+
export PSI_VIDEO_DEMO_ROOT=${PSI_VIDEO_DEMO_ROOT:-/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/videogen/motion_example_0506_demo}
|
| 8 |
+
|
| 9 |
+
export BUILD_CLIPS=1
|
| 10 |
+
export CACHE_EMBEDS=1
|
| 11 |
+
export BUILD_DANCE=0
|
| 12 |
+
export CACHE_DANCE=0
|
| 13 |
+
export BUILD_MOTION_EXAMPLE=1
|
| 14 |
+
export CACHE_MOTION_EXAMPLE=1
|
| 15 |
+
|
| 16 |
+
export BUILD_NUM_WORKERS=${BUILD_NUM_WORKERS:-160}
|
| 17 |
+
export EMBED_NUM_WORKERS=${EMBED_NUM_WORKERS:-160}
|
| 18 |
+
export OVERWRITE_CLIPS=${OVERWRITE_CLIPS:-1}
|
| 19 |
+
export OVERWRITE_EMBEDS=${OVERWRITE_EMBEDS:-1}
|
| 20 |
+
export OMP_NUM_THREADS=${OMP_NUM_THREADS:-1}
|
| 21 |
+
|
| 22 |
+
exec bash cache_video_omni_embeds.sh
|
cache_omni_hidden_states_v20.sh
ADDED
|
@@ -0,0 +1,78 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#!/usr/bin/env bash
|
| 2 |
+
set -euo pipefail
|
| 3 |
+
|
| 4 |
+
ROOT_DIR="/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/Psi0-yth-v20"
|
| 5 |
+
PY_SCRIPT="$ROOT_DIR/scripts/data/cache_omni_hidden_states.py"
|
| 6 |
+
PYTHON_BIN="/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/envs/Psi0/bin/python"
|
| 7 |
+
MODEL_PATH="/cpfs01/VideoGen/public_ckpt/Qwen/Qwen2.5-Omni-3B"
|
| 8 |
+
OUTPUT_ROOT="${OUTPUT_ROOT:-/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/videogen/psi0_omni_hidden_states_v20}"
|
| 9 |
+
TASKS="${TASKS:-audio_only,audio_command_text,audio_command_only,video_clip_text,video_clip_only}"
|
| 10 |
+
SOURCES="${SOURCES:-2264_with_audio,psi0_dance_audio,qwen3_tts_seed_audio_command,dance_1723_video_10s_clip50,motion_example_0506_demo_clip50,2264_with_audio_clip50}"
|
| 11 |
+
MODE="${1:-${MODE:-estimate}}"
|
| 12 |
+
shift || true
|
| 13 |
+
|
| 14 |
+
if [[ ! -f "$PY_SCRIPT" ]]; then
|
| 15 |
+
echo "Missing script: $PY_SCRIPT" >&2
|
| 16 |
+
exit 1
|
| 17 |
+
fi
|
| 18 |
+
|
| 19 |
+
export PYTHONPATH="$ROOT_DIR/src:${PYTHONPATH:-}"
|
| 20 |
+
export TOKENIZERS_PARALLELISM="${TOKENIZERS_PARALLELISM:-false}"
|
| 21 |
+
export NCCL_DEBUG="${NCCL_DEBUG:-WARN}"
|
| 22 |
+
export OMP_NUM_THREADS="${OMP_NUM_THREADS:-4}"
|
| 23 |
+
export PYTORCH_CUDA_ALLOC_CONF="${PYTORCH_CUDA_ALLOC_CONF:-expandable_segments:True}"
|
| 24 |
+
|
| 25 |
+
COMMON_ARGS=(
|
| 26 |
+
"$PY_SCRIPT"
|
| 27 |
+
--mode "$MODE"
|
| 28 |
+
--model-path "$MODEL_PATH"
|
| 29 |
+
--output-root "$OUTPUT_ROOT"
|
| 30 |
+
--tasks "$TASKS"
|
| 31 |
+
--sources "$SOURCES"
|
| 32 |
+
)
|
| 33 |
+
|
| 34 |
+
if [[ "$MODE" == "estimate" ]]; then
|
| 35 |
+
echo "[v20] mode=$MODE"
|
| 36 |
+
echo "[v20] tasks=$TASKS"
|
| 37 |
+
echo "[v20] sources=$SOURCES"
|
| 38 |
+
echo "[v20] output_root=$OUTPUT_ROOT"
|
| 39 |
+
exec "$PYTHON_BIN" "${COMMON_ARGS[@]}" "$@"
|
| 40 |
+
fi
|
| 41 |
+
|
| 42 |
+
if ! command -v nvidia-smi >/dev/null 2>&1; then
|
| 43 |
+
echo "nvidia-smi not found; benchmark/export needs GPU" >&2
|
| 44 |
+
exit 1
|
| 45 |
+
fi
|
| 46 |
+
|
| 47 |
+
if [[ -n "${CUDA_VISIBLE_DEVICES:-}" ]]; then
|
| 48 |
+
IFS=',' read -r -a _visible_gpu_ids <<< "$CUDA_VISIBLE_DEVICES"
|
| 49 |
+
VISIBLE_GPU_COUNT=0
|
| 50 |
+
for _gpu_id in "${_visible_gpu_ids[@]}"; do
|
| 51 |
+
if [[ -n "$_gpu_id" ]]; then
|
| 52 |
+
VISIBLE_GPU_COUNT=$((VISIBLE_GPU_COUNT + 1))
|
| 53 |
+
fi
|
| 54 |
+
done
|
| 55 |
+
else
|
| 56 |
+
VISIBLE_GPU_COUNT="$(nvidia-smi --query-gpu=index --format=csv,noheader | wc -l | tr -d ' ')"
|
| 57 |
+
fi
|
| 58 |
+
|
| 59 |
+
if [[ -z "${NPROC:-}" ]]; then
|
| 60 |
+
NPROC="$VISIBLE_GPU_COUNT"
|
| 61 |
+
fi
|
| 62 |
+
|
| 63 |
+
if [[ "$MODE" != "benchmark" && "$MODE" != "export" ]]; then
|
| 64 |
+
echo "Unsupported mode: $MODE" >&2
|
| 65 |
+
exit 1
|
| 66 |
+
fi
|
| 67 |
+
|
| 68 |
+
echo "[v20] mode=$MODE"
|
| 69 |
+
echo "[v20] nproc=$NPROC"
|
| 70 |
+
echo "[v20] tasks=$TASKS"
|
| 71 |
+
echo "[v20] sources=$SOURCES"
|
| 72 |
+
echo "[v20] output_root=$OUTPUT_ROOT"
|
| 73 |
+
|
| 74 |
+
exec "$PYTHON_BIN" -m torch.distributed.run \
|
| 75 |
+
--standalone \
|
| 76 |
+
--nproc_per_node="$NPROC" \
|
| 77 |
+
"${COMMON_ARGS[@]}" \
|
| 78 |
+
"$@"
|
cache_video_omni_embeds.py
ADDED
|
@@ -0,0 +1,996 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from __future__ import annotations
|
| 2 |
+
|
| 3 |
+
import argparse
|
| 4 |
+
import hashlib
|
| 5 |
+
import json
|
| 6 |
+
import os
|
| 7 |
+
import socket
|
| 8 |
+
import time
|
| 9 |
+
import uuid
|
| 10 |
+
from concurrent.futures import FIRST_COMPLETED, ProcessPoolExecutor, ThreadPoolExecutor, as_completed, wait
|
| 11 |
+
from itertools import islice
|
| 12 |
+
from pathlib import Path
|
| 13 |
+
|
| 14 |
+
import numpy as np
|
| 15 |
+
from tqdm import tqdm
|
| 16 |
+
from transformers import Qwen2_5OmniProcessor
|
| 17 |
+
|
| 18 |
+
from psi.data.seed.seed_video_motion_dataset import (
|
| 19 |
+
load_video_array,
|
| 20 |
+
load_video_clip_array,
|
| 21 |
+
video_embed_cache_path,
|
| 22 |
+
)
|
| 23 |
+
|
| 24 |
+
|
| 25 |
+
REQUIRED_CACHE_KEYS = (
|
| 26 |
+
"pixel_values_videos",
|
| 27 |
+
"video_grid_thw",
|
| 28 |
+
"video_second_per_grid",
|
| 29 |
+
"video_sample_fps",
|
| 30 |
+
"video_size_pixels",
|
| 31 |
+
"source_video_fps",
|
| 32 |
+
"video_duration_sec",
|
| 33 |
+
)
|
| 34 |
+
|
| 35 |
+
_WORKER_PROCESSOR = None
|
| 36 |
+
_WORKER_MODEL_PATH = ""
|
| 37 |
+
_SOURCE_PATH_EXISTS: dict[str, bool] = {}
|
| 38 |
+
|
| 39 |
+
|
| 40 |
+
def parse_args() -> argparse.Namespace:
|
| 41 |
+
parser = argparse.ArgumentParser(description="Cache Qwen2.5-Omni video processor outputs.")
|
| 42 |
+
parser.add_argument("--video-root", type=str, default="")
|
| 43 |
+
parser.add_argument("--dataset-root", type=str, default="")
|
| 44 |
+
parser.add_argument("--metadata-jsonl", type=str, default="")
|
| 45 |
+
parser.add_argument("--video-path-key", type=str, default="video_path")
|
| 46 |
+
parser.add_argument("--embed-root", type=str, required=True)
|
| 47 |
+
parser.add_argument("--model-path", type=str, required=True)
|
| 48 |
+
parser.add_argument("--video-sample-fps", type=float, default=1.0)
|
| 49 |
+
parser.add_argument("--video-size-pixels", type=int, default=50176)
|
| 50 |
+
parser.add_argument("--video-decoder", type=str, default="ffmpeg", choices=["opencv", "ffmpeg"])
|
| 51 |
+
parser.add_argument("--dtype", type=str, default="float16", choices=["float16", "float32"])
|
| 52 |
+
parser.add_argument("--num-workers", type=int, default=1)
|
| 53 |
+
parser.add_argument("--save-compressed", action="store_true", default=False)
|
| 54 |
+
parser.add_argument(
|
| 55 |
+
"--existing-cache-check",
|
| 56 |
+
choices=["strict", "size"],
|
| 57 |
+
default=os.environ.get("PSI_EXISTING_CACHE_CHECK", "strict"),
|
| 58 |
+
help="How to validate existing npz caches before skipping. strict opens npz; size only checks final file size.",
|
| 59 |
+
)
|
| 60 |
+
parser.add_argument("--limit", type=int, default=0)
|
| 61 |
+
parser.add_argument("--overwrite", action="store_true", default=False)
|
| 62 |
+
parser.add_argument("--shard-num", type=int, default=1)
|
| 63 |
+
parser.add_argument("--shard-id", type=int, default=0)
|
| 64 |
+
parser.add_argument("--input-already-sharded", action="store_true", default=False)
|
| 65 |
+
parser.add_argument(
|
| 66 |
+
"--direct-source-clips",
|
| 67 |
+
action="store_true",
|
| 68 |
+
default=False,
|
| 69 |
+
help=(
|
| 70 |
+
"Read source_video_path + chunk_start_sec/duration from metadata, but write cache "
|
| 71 |
+
"to the canonical path derived from video_path."
|
| 72 |
+
),
|
| 73 |
+
)
|
| 74 |
+
return parser.parse_args()
|
| 75 |
+
|
| 76 |
+
|
| 77 |
+
def _resolve_path(path_value: str, base_dir: Path) -> Path:
|
| 78 |
+
path = Path(str(path_value))
|
| 79 |
+
return path if path.is_absolute() else base_dir / path
|
| 80 |
+
|
| 81 |
+
|
| 82 |
+
def resolve_metadata_context(args: argparse.Namespace) -> tuple[Path, Path]:
|
| 83 |
+
dataset_root = Path(args.dataset_root) if args.dataset_root else Path(args.metadata_jsonl).parent
|
| 84 |
+
metadata_path = Path(args.metadata_jsonl)
|
| 85 |
+
if not metadata_path.is_absolute():
|
| 86 |
+
metadata_path = dataset_root / metadata_path
|
| 87 |
+
if not metadata_path.exists():
|
| 88 |
+
raise FileNotFoundError(f"metadata_jsonl does not exist: {metadata_path}")
|
| 89 |
+
return dataset_root, metadata_path
|
| 90 |
+
|
| 91 |
+
|
| 92 |
+
def build_video_task_from_row(
|
| 93 |
+
row: dict,
|
| 94 |
+
args: argparse.Namespace,
|
| 95 |
+
dataset_root: Path,
|
| 96 |
+
metadata_path: Path,
|
| 97 |
+
line_no: int,
|
| 98 |
+
) -> dict:
|
| 99 |
+
video_value = row.get(args.video_path_key) or row.get("video_path") or row.get("original_video")
|
| 100 |
+
if not video_value:
|
| 101 |
+
raise ValueError(f"Missing video path at {metadata_path}:{line_no}")
|
| 102 |
+
video_path = _resolve_path(str(video_value), dataset_root)
|
| 103 |
+
|
| 104 |
+
task = {"video_path": video_path}
|
| 105 |
+
if args.direct_source_clips:
|
| 106 |
+
source_value = row.get("source_video_path")
|
| 107 |
+
if not source_value:
|
| 108 |
+
raise ValueError(f"Missing source_video_path at {metadata_path}:{line_no}")
|
| 109 |
+
source_video_path = _resolve_path(str(source_value), dataset_root)
|
| 110 |
+
source_key = str(source_video_path)
|
| 111 |
+
exists = _SOURCE_PATH_EXISTS.get(source_key)
|
| 112 |
+
if exists is None:
|
| 113 |
+
exists = source_video_path.exists()
|
| 114 |
+
_SOURCE_PATH_EXISTS[source_key] = exists
|
| 115 |
+
if not exists:
|
| 116 |
+
raise FileNotFoundError(f"Source video file does not exist: {source_video_path}")
|
| 117 |
+
task.update(
|
| 118 |
+
{
|
| 119 |
+
"decode_video_path": source_video_path,
|
| 120 |
+
"clip_start_sec": float(row.get("chunk_start_sec") or 0.0),
|
| 121 |
+
"clip_duration_sec": float(row.get("duration") or row.get("video_duration_sec") or 0.0),
|
| 122 |
+
}
|
| 123 |
+
)
|
| 124 |
+
else:
|
| 125 |
+
if not video_path.exists():
|
| 126 |
+
raise FileNotFoundError(f"Video file does not exist: {video_path}")
|
| 127 |
+
task["decode_video_path"] = video_path
|
| 128 |
+
return task
|
| 129 |
+
|
| 130 |
+
|
| 131 |
+
def iter_metadata_tasks_stream(
|
| 132 |
+
args: argparse.Namespace,
|
| 133 |
+
start_index: int = 0,
|
| 134 |
+
):
|
| 135 |
+
dataset_root, metadata_path = resolve_metadata_context(args)
|
| 136 |
+
stop_index = int(args.limit) if args.limit > 0 else None
|
| 137 |
+
with metadata_path.open("r", encoding="utf-8") as f:
|
| 138 |
+
iterator = islice(f, start_index, None) if start_index > 0 else f
|
| 139 |
+
for task_index, line in enumerate(iterator, start=start_index):
|
| 140 |
+
if stop_index is not None and task_index >= stop_index:
|
| 141 |
+
break
|
| 142 |
+
line = line.strip()
|
| 143 |
+
if not line:
|
| 144 |
+
continue
|
| 145 |
+
row = json.loads(line)
|
| 146 |
+
task = build_video_task_from_row(
|
| 147 |
+
row=row,
|
| 148 |
+
args=args,
|
| 149 |
+
dataset_root=dataset_root,
|
| 150 |
+
metadata_path=metadata_path,
|
| 151 |
+
line_no=task_index + 1,
|
| 152 |
+
)
|
| 153 |
+
task["task_index"] = task_index
|
| 154 |
+
yield task
|
| 155 |
+
|
| 156 |
+
|
| 157 |
+
def iter_video_tasks(args: argparse.Namespace) -> list[dict]:
|
| 158 |
+
if args.metadata_jsonl:
|
| 159 |
+
dataset_root, metadata_path = resolve_metadata_context(args)
|
| 160 |
+
|
| 161 |
+
tasks = []
|
| 162 |
+
seen = set()
|
| 163 |
+
with metadata_path.open("r", encoding="utf-8") as f:
|
| 164 |
+
for line_no, line in enumerate(f, start=1):
|
| 165 |
+
line = line.strip()
|
| 166 |
+
if not line:
|
| 167 |
+
continue
|
| 168 |
+
row = json.loads(line)
|
| 169 |
+
task = build_video_task_from_row(
|
| 170 |
+
row=row,
|
| 171 |
+
args=args,
|
| 172 |
+
dataset_root=dataset_root,
|
| 173 |
+
metadata_path=metadata_path,
|
| 174 |
+
line_no=line_no,
|
| 175 |
+
)
|
| 176 |
+
video_path = task["video_path"]
|
| 177 |
+
|
| 178 |
+
key = str(video_embed_cache_path(
|
| 179 |
+
video_path,
|
| 180 |
+
args.embed_root,
|
| 181 |
+
video_sample_fps=args.video_sample_fps,
|
| 182 |
+
video_size_pixels=args.video_size_pixels,
|
| 183 |
+
))
|
| 184 |
+
if key not in seen:
|
| 185 |
+
seen.add(key)
|
| 186 |
+
tasks.append(task)
|
| 187 |
+
return tasks
|
| 188 |
+
|
| 189 |
+
if not args.video_root:
|
| 190 |
+
raise ValueError("Either --metadata-jsonl or --video-root is required")
|
| 191 |
+
video_root = Path(args.video_root)
|
| 192 |
+
if not video_root.exists():
|
| 193 |
+
raise FileNotFoundError(f"video_root does not exist: {video_root}")
|
| 194 |
+
return [{"video_path": path, "decode_video_path": path} for path in sorted(video_root.glob("*.mp4"))]
|
| 195 |
+
|
| 196 |
+
|
| 197 |
+
def is_valid_cache(path: Path) -> bool:
|
| 198 |
+
if not path.exists() or path.stat().st_size <= 0:
|
| 199 |
+
return False
|
| 200 |
+
try:
|
| 201 |
+
with np.load(path, allow_pickle=False) as cache:
|
| 202 |
+
for key in REQUIRED_CACHE_KEYS:
|
| 203 |
+
if key not in cache:
|
| 204 |
+
return False
|
| 205 |
+
pixel_values = cache["pixel_values_videos"]
|
| 206 |
+
grid = cache["video_grid_thw"]
|
| 207 |
+
if pixel_values.size <= 0 or grid.size <= 0:
|
| 208 |
+
return False
|
| 209 |
+
except Exception:
|
| 210 |
+
return False
|
| 211 |
+
return True
|
| 212 |
+
|
| 213 |
+
|
| 214 |
+
def is_existing_cache_ready(path: Path, check_mode: str) -> bool:
|
| 215 |
+
if not path.exists() or path.stat().st_size <= 0:
|
| 216 |
+
return False
|
| 217 |
+
if check_mode == "size":
|
| 218 |
+
return True
|
| 219 |
+
return is_valid_cache(path)
|
| 220 |
+
|
| 221 |
+
|
| 222 |
+
def is_existing_cache_ready_fast(path: Path, check_mode: str) -> bool:
|
| 223 |
+
if check_mode == "size":
|
| 224 |
+
try:
|
| 225 |
+
return path.stat().st_size > 0
|
| 226 |
+
except FileNotFoundError:
|
| 227 |
+
return False
|
| 228 |
+
return is_existing_cache_ready(path, check_mode)
|
| 229 |
+
|
| 230 |
+
|
| 231 |
+
def remove_path(path: Path) -> None:
|
| 232 |
+
try:
|
| 233 |
+
path.unlink()
|
| 234 |
+
except FileNotFoundError:
|
| 235 |
+
pass
|
| 236 |
+
|
| 237 |
+
|
| 238 |
+
def stable_hash_mod(value: str, modulo: int) -> int:
|
| 239 |
+
digest = hashlib.sha1(value.encode("utf-8")).digest()
|
| 240 |
+
return int.from_bytes(digest[:8], "big") % int(modulo)
|
| 241 |
+
|
| 242 |
+
|
| 243 |
+
def shard_manifest_path(embed_root: Path, shard_id: int, shard_num: int) -> Path:
|
| 244 |
+
return embed_root / f"manifest.shard-{shard_id:05d}-of-{shard_num:05d}.json"
|
| 245 |
+
|
| 246 |
+
|
| 247 |
+
def resume_state_path(embed_root: Path, shard_id: int, shard_num: int) -> Path:
|
| 248 |
+
if shard_num > 1:
|
| 249 |
+
return embed_root / f"resume.shard-{shard_id:05d}-of-{shard_num:05d}.json"
|
| 250 |
+
return embed_root / "resume.json"
|
| 251 |
+
|
| 252 |
+
|
| 253 |
+
def write_json_atomic(path: Path, payload: dict) -> None:
|
| 254 |
+
tmp_path = path.with_name(f".{path.name}.{os.getpid()}.{uuid.uuid4().hex}.tmp")
|
| 255 |
+
tmp_path.write_text(
|
| 256 |
+
json.dumps(payload, ensure_ascii=False, indent=2),
|
| 257 |
+
encoding="utf-8",
|
| 258 |
+
)
|
| 259 |
+
tmp_path.replace(path)
|
| 260 |
+
|
| 261 |
+
|
| 262 |
+
def merge_shard_manifests(embed_root: Path, shard_num: int) -> bool:
|
| 263 |
+
shard_paths = [shard_manifest_path(embed_root, shard_id, shard_num) for shard_id in range(shard_num)]
|
| 264 |
+
if any(not path.exists() for path in shard_paths):
|
| 265 |
+
return False
|
| 266 |
+
|
| 267 |
+
manifests = []
|
| 268 |
+
for path in shard_paths:
|
| 269 |
+
try:
|
| 270 |
+
manifests.append(json.loads(path.read_text(encoding="utf-8")))
|
| 271 |
+
except Exception:
|
| 272 |
+
return False
|
| 273 |
+
if not manifests:
|
| 274 |
+
return False
|
| 275 |
+
|
| 276 |
+
merged = dict(manifests[0])
|
| 277 |
+
merged["count"] = int(sum(int(item.get("count") or 0) for item in manifests))
|
| 278 |
+
if all(bool(item.get("input_already_sharded")) for item in manifests):
|
| 279 |
+
merged["total_count"] = int(sum(int(item.get("count") or 0) for item in manifests))
|
| 280 |
+
else:
|
| 281 |
+
merged["total_count"] = int(max(int(item.get("total_count") or 0) for item in manifests))
|
| 282 |
+
merged["shard_id"] = None
|
| 283 |
+
merged["shard_num"] = int(shard_num)
|
| 284 |
+
merged["shards"] = [path.name for path in shard_paths]
|
| 285 |
+
write_json_atomic(embed_root / "manifest.json", merged)
|
| 286 |
+
return True
|
| 287 |
+
|
| 288 |
+
|
| 289 |
+
def maybe_load_json(path: Path) -> dict | None:
|
| 290 |
+
if not path.exists():
|
| 291 |
+
return None
|
| 292 |
+
try:
|
| 293 |
+
return json.loads(path.read_text(encoding="utf-8"))
|
| 294 |
+
except Exception:
|
| 295 |
+
return None
|
| 296 |
+
|
| 297 |
+
|
| 298 |
+
def load_known_total_count(args: argparse.Namespace, embed_root: Path) -> int | None:
|
| 299 |
+
manifest_path = (
|
| 300 |
+
shard_manifest_path(embed_root, args.shard_id, args.shard_num)
|
| 301 |
+
if args.shard_num > 1
|
| 302 |
+
else embed_root / "manifest.json"
|
| 303 |
+
)
|
| 304 |
+
payload = maybe_load_json(manifest_path)
|
| 305 |
+
if payload and str(payload.get("metadata_jsonl")) == str(args.metadata_jsonl):
|
| 306 |
+
count = int(payload.get("count") or 0)
|
| 307 |
+
if count > 0:
|
| 308 |
+
return min(count, int(args.limit)) if args.limit > 0 else count
|
| 309 |
+
return None
|
| 310 |
+
|
| 311 |
+
|
| 312 |
+
def count_metadata_rows(args: argparse.Namespace) -> int:
|
| 313 |
+
_, metadata_path = resolve_metadata_context(args)
|
| 314 |
+
count = 0
|
| 315 |
+
with metadata_path.open("rb") as f:
|
| 316 |
+
for chunk in iter(lambda: f.read(1024 * 1024 * 16), b""):
|
| 317 |
+
count += chunk.count(b"\n")
|
| 318 |
+
if args.limit > 0:
|
| 319 |
+
return min(count, int(args.limit))
|
| 320 |
+
return count
|
| 321 |
+
|
| 322 |
+
|
| 323 |
+
def dense_source_key(row: dict) -> str:
|
| 324 |
+
for key in ("name", "id", "video_path"):
|
| 325 |
+
value = row.get(key)
|
| 326 |
+
if isinstance(value, str) and value:
|
| 327 |
+
return value.split(".chunk", 1)[0]
|
| 328 |
+
return ""
|
| 329 |
+
|
| 330 |
+
|
| 331 |
+
def cache_name_from_row(row: dict, args: argparse.Namespace) -> str:
|
| 332 |
+
video_value = row.get(args.video_path_key) or row.get("video_path") or row.get("original_video")
|
| 333 |
+
fps_tag = str(float(args.video_sample_fps)).replace(".", "p")
|
| 334 |
+
return f"{Path(str(video_value)).stem}.fps{fps_tag}.size{int(args.video_size_pixels)}.npz"
|
| 335 |
+
|
| 336 |
+
|
| 337 |
+
def compatible_resume_payload(
|
| 338 |
+
payload: dict | None,
|
| 339 |
+
args: argparse.Namespace,
|
| 340 |
+
embed_root: Path,
|
| 341 |
+
*,
|
| 342 |
+
shard_id: int,
|
| 343 |
+
shard_num: int,
|
| 344 |
+
) -> bool:
|
| 345 |
+
if not payload:
|
| 346 |
+
return False
|
| 347 |
+
expected = {
|
| 348 |
+
"dataset_root": str(args.dataset_root),
|
| 349 |
+
"embed_root": str(embed_root),
|
| 350 |
+
"video_sample_fps": float(args.video_sample_fps),
|
| 351 |
+
"video_size_pixels": int(args.video_size_pixels),
|
| 352 |
+
"shard_id": int(shard_id),
|
| 353 |
+
"shard_num": int(shard_num),
|
| 354 |
+
"input_already_sharded": bool(args.input_already_sharded),
|
| 355 |
+
"direct_source_clips": bool(args.direct_source_clips),
|
| 356 |
+
}
|
| 357 |
+
return all(payload.get(key) == value for key, value in expected.items())
|
| 358 |
+
|
| 359 |
+
|
| 360 |
+
def parse_resume_state_name(path: Path) -> tuple[int, int] | None:
|
| 361 |
+
name = path.name
|
| 362 |
+
prefix = "resume.shard-"
|
| 363 |
+
if not name.startswith(prefix) or "-of-" not in name or not name.endswith(".json"):
|
| 364 |
+
return None
|
| 365 |
+
try:
|
| 366 |
+
left, right = name[len(prefix):-len(".json")].split("-of-", 1)
|
| 367 |
+
return int(left), int(right)
|
| 368 |
+
except ValueError:
|
| 369 |
+
return None
|
| 370 |
+
|
| 371 |
+
|
| 372 |
+
def infer_frontier_from_parent_resume(args: argparse.Namespace, embed_root: Path) -> int:
|
| 373 |
+
if args.shard_num <= 1:
|
| 374 |
+
return 0
|
| 375 |
+
dataset_root, _ = resolve_metadata_context(args)
|
| 376 |
+
candidates = []
|
| 377 |
+
for resume_path in sorted(embed_root.glob("resume.shard-*-of-*.json")):
|
| 378 |
+
parsed = parse_resume_state_name(resume_path)
|
| 379 |
+
if parsed is None:
|
| 380 |
+
continue
|
| 381 |
+
parent_id, parent_num = parsed
|
| 382 |
+
if parent_num == args.shard_num:
|
| 383 |
+
continue
|
| 384 |
+
payload = maybe_load_json(resume_path)
|
| 385 |
+
if not compatible_resume_payload(
|
| 386 |
+
payload,
|
| 387 |
+
args,
|
| 388 |
+
embed_root,
|
| 389 |
+
shard_id=parent_id,
|
| 390 |
+
shard_num=parent_num,
|
| 391 |
+
):
|
| 392 |
+
continue
|
| 393 |
+
parent_frontier = max(0, int(payload.get("frontier") or 0))
|
| 394 |
+
if parent_frontier <= 0:
|
| 395 |
+
continue
|
| 396 |
+
parent_metadata = dataset_root / str(payload.get("metadata_jsonl") or "")
|
| 397 |
+
if not parent_metadata.exists():
|
| 398 |
+
parent_metadata = dataset_root / f"metadata.shard-{parent_id:05d}-of-{parent_num:05d}.jsonl"
|
| 399 |
+
if not parent_metadata.exists():
|
| 400 |
+
continue
|
| 401 |
+
candidates.append((parent_num, parent_id, parent_frontier, parent_metadata))
|
| 402 |
+
|
| 403 |
+
if not candidates:
|
| 404 |
+
return 0
|
| 405 |
+
|
| 406 |
+
completed_cache_names = set()
|
| 407 |
+
for parent_num, parent_id, parent_frontier, parent_metadata in candidates:
|
| 408 |
+
with parent_metadata.open("r", encoding="utf-8") as f:
|
| 409 |
+
iterator = tqdm(
|
| 410 |
+
islice(f, parent_frontier),
|
| 411 |
+
total=parent_frontier,
|
| 412 |
+
desc=f"load resume {parent_id}/{parent_num}",
|
| 413 |
+
)
|
| 414 |
+
for line in iterator:
|
| 415 |
+
line = line.strip()
|
| 416 |
+
if not line:
|
| 417 |
+
continue
|
| 418 |
+
completed_cache_names.add(cache_name_from_row(json.loads(line), args))
|
| 419 |
+
|
| 420 |
+
inherited = 0
|
| 421 |
+
for video_task in tqdm(iter_metadata_tasks_stream(args), desc="inherit resume"):
|
| 422 |
+
cache_name = video_embed_cache_path(
|
| 423 |
+
Path(video_task["video_path"]),
|
| 424 |
+
embed_root,
|
| 425 |
+
video_sample_fps=args.video_sample_fps,
|
| 426 |
+
video_size_pixels=args.video_size_pixels,
|
| 427 |
+
).name
|
| 428 |
+
if cache_name not in completed_cache_names:
|
| 429 |
+
break
|
| 430 |
+
inherited = int(video_task["task_index"]) + 1
|
| 431 |
+
|
| 432 |
+
print(
|
| 433 |
+
f"[RESUME] inherited frontier={inherited} from "
|
| 434 |
+
f"{len(candidates)} compatible previous shard resume file(s)"
|
| 435 |
+
)
|
| 436 |
+
return inherited
|
| 437 |
+
|
| 438 |
+
|
| 439 |
+
def write_manifest(
|
| 440 |
+
args: argparse.Namespace,
|
| 441 |
+
embed_root: Path,
|
| 442 |
+
total_video_paths: int,
|
| 443 |
+
) -> None:
|
| 444 |
+
manifest = {
|
| 445 |
+
"video_root": str(args.video_root),
|
| 446 |
+
"dataset_root": str(args.dataset_root),
|
| 447 |
+
"metadata_jsonl": str(args.metadata_jsonl),
|
| 448 |
+
"embed_root": str(embed_root),
|
| 449 |
+
"model_path": str(args.model_path),
|
| 450 |
+
"video_sample_fps": float(args.video_sample_fps),
|
| 451 |
+
"video_size_pixels": int(args.video_size_pixels),
|
| 452 |
+
"video_decoder": str(args.video_decoder),
|
| 453 |
+
"dtype": args.dtype,
|
| 454 |
+
"num_workers": int(args.num_workers),
|
| 455 |
+
"save_compressed": bool(args.save_compressed),
|
| 456 |
+
"existing_cache_check": str(args.existing_cache_check),
|
| 457 |
+
"count": int(total_video_paths),
|
| 458 |
+
"total_count": int(total_video_paths),
|
| 459 |
+
"shard_id": int(args.shard_id),
|
| 460 |
+
"shard_num": int(args.shard_num),
|
| 461 |
+
"input_already_sharded": bool(args.input_already_sharded),
|
| 462 |
+
}
|
| 463 |
+
manifest_path = (
|
| 464 |
+
shard_manifest_path(embed_root, args.shard_id, args.shard_num)
|
| 465 |
+
if args.shard_num > 1
|
| 466 |
+
else embed_root / "manifest.json"
|
| 467 |
+
)
|
| 468 |
+
write_json_atomic(manifest_path, manifest)
|
| 469 |
+
if args.shard_num > 1:
|
| 470 |
+
merge_shard_manifests(embed_root, args.shard_num)
|
| 471 |
+
|
| 472 |
+
|
| 473 |
+
def fast_resume_enabled(args: argparse.Namespace) -> bool:
|
| 474 |
+
return bool(args.metadata_jsonl and args.input_already_sharded and args.direct_source_clips)
|
| 475 |
+
|
| 476 |
+
|
| 477 |
+
def load_resume_frontier(args: argparse.Namespace, embed_root: Path) -> tuple[int, int | None, Path]:
|
| 478 |
+
state_path = resume_state_path(embed_root, args.shard_id, args.shard_num)
|
| 479 |
+
payload = maybe_load_json(state_path)
|
| 480 |
+
if not compatible_resume_payload(
|
| 481 |
+
payload,
|
| 482 |
+
args,
|
| 483 |
+
embed_root,
|
| 484 |
+
shard_id=args.shard_id,
|
| 485 |
+
shard_num=args.shard_num,
|
| 486 |
+
) or not payload or payload.get("metadata_jsonl") != str(args.metadata_jsonl):
|
| 487 |
+
return infer_frontier_from_parent_resume(args, embed_root), None, state_path
|
| 488 |
+
frontier = max(0, int(payload.get("frontier") or 0))
|
| 489 |
+
total_count = int(payload.get("total_count") or 0) or None
|
| 490 |
+
if args.limit > 0:
|
| 491 |
+
frontier = min(frontier, int(args.limit))
|
| 492 |
+
if total_count is not None:
|
| 493 |
+
total_count = min(total_count, int(args.limit))
|
| 494 |
+
return frontier, total_count, state_path
|
| 495 |
+
|
| 496 |
+
|
| 497 |
+
class ResumeTracker:
|
| 498 |
+
def __init__(
|
| 499 |
+
self,
|
| 500 |
+
*,
|
| 501 |
+
path: Path,
|
| 502 |
+
args: argparse.Namespace,
|
| 503 |
+
embed_root: Path,
|
| 504 |
+
frontier: int,
|
| 505 |
+
total_count: int | None,
|
| 506 |
+
) -> None:
|
| 507 |
+
self.path = path
|
| 508 |
+
self.args = args
|
| 509 |
+
self.embed_root = embed_root
|
| 510 |
+
self.frontier = max(0, int(frontier))
|
| 511 |
+
self.total_count = int(total_count) if total_count is not None else None
|
| 512 |
+
self._completed: set[int] = set()
|
| 513 |
+
self._dirty = True
|
| 514 |
+
self._last_flush_time = 0.0
|
| 515 |
+
|
| 516 |
+
def mark_complete(self, task_index: int | None) -> None:
|
| 517 |
+
if task_index is None or task_index < 0 or task_index < self.frontier:
|
| 518 |
+
return
|
| 519 |
+
self._completed.add(int(task_index))
|
| 520 |
+
advanced = False
|
| 521 |
+
while self.frontier in self._completed:
|
| 522 |
+
self._completed.remove(self.frontier)
|
| 523 |
+
self.frontier += 1
|
| 524 |
+
advanced = True
|
| 525 |
+
if advanced:
|
| 526 |
+
self._dirty = True
|
| 527 |
+
|
| 528 |
+
def maybe_set_total_count(self, total_count: int) -> None:
|
| 529 |
+
total_count = int(total_count)
|
| 530 |
+
if self.total_count != total_count:
|
| 531 |
+
self.total_count = total_count
|
| 532 |
+
self._dirty = True
|
| 533 |
+
|
| 534 |
+
def flush(self, force: bool = False) -> None:
|
| 535 |
+
now = time.time()
|
| 536 |
+
if not force and (not self._dirty or now - self._last_flush_time < 10.0):
|
| 537 |
+
return
|
| 538 |
+
payload = {
|
| 539 |
+
"dataset_root": str(self.args.dataset_root),
|
| 540 |
+
"metadata_jsonl": str(self.args.metadata_jsonl),
|
| 541 |
+
"embed_root": str(self.embed_root),
|
| 542 |
+
"video_sample_fps": float(self.args.video_sample_fps),
|
| 543 |
+
"video_size_pixels": int(self.args.video_size_pixels),
|
| 544 |
+
"shard_id": int(self.args.shard_id),
|
| 545 |
+
"shard_num": int(self.args.shard_num),
|
| 546 |
+
"input_already_sharded": bool(self.args.input_already_sharded),
|
| 547 |
+
"direct_source_clips": bool(self.args.direct_source_clips),
|
| 548 |
+
"frontier": int(self.frontier),
|
| 549 |
+
"total_count": int(self.total_count) if self.total_count is not None else None,
|
| 550 |
+
"updated_at": now,
|
| 551 |
+
}
|
| 552 |
+
write_json_atomic(self.path, payload)
|
| 553 |
+
self._dirty = False
|
| 554 |
+
self._last_flush_time = now
|
| 555 |
+
|
| 556 |
+
|
| 557 |
+
def extend_resume_frontier(
|
| 558 |
+
args: argparse.Namespace,
|
| 559 |
+
embed_root: Path,
|
| 560 |
+
frontier: int,
|
| 561 |
+
) -> int:
|
| 562 |
+
if args.overwrite:
|
| 563 |
+
return 0
|
| 564 |
+
current_frontier = max(0, int(frontier))
|
| 565 |
+
check_workers = max(1, int(os.environ.get("RESUME_SCAN_NUM_WORKERS", "32")))
|
| 566 |
+
batch_size = max(check_workers, int(os.environ.get("RESUME_SCAN_BATCH_SIZE", str(check_workers * 8))))
|
| 567 |
+
iterator = iter_metadata_tasks_stream(args, start_index=current_frontier)
|
| 568 |
+
progress = tqdm(desc=f"resume frontier x{check_workers}")
|
| 569 |
+
|
| 570 |
+
def check_task(video_task: dict) -> tuple[int, bool, Path]:
|
| 571 |
+
video_path = Path(video_task["video_path"])
|
| 572 |
+
cache_path = video_embed_cache_path(
|
| 573 |
+
video_path,
|
| 574 |
+
embed_root,
|
| 575 |
+
video_sample_fps=args.video_sample_fps,
|
| 576 |
+
video_size_pixels=args.video_size_pixels,
|
| 577 |
+
)
|
| 578 |
+
return (
|
| 579 |
+
int(video_task["task_index"]),
|
| 580 |
+
is_existing_cache_ready_fast(cache_path, args.existing_cache_check),
|
| 581 |
+
cache_path,
|
| 582 |
+
)
|
| 583 |
+
|
| 584 |
+
with ThreadPoolExecutor(max_workers=check_workers) as executor:
|
| 585 |
+
while True:
|
| 586 |
+
batch = list(islice(iterator, batch_size))
|
| 587 |
+
if not batch:
|
| 588 |
+
break
|
| 589 |
+
results = list(executor.map(check_task, batch))
|
| 590 |
+
for task_index, ready, cache_path in results:
|
| 591 |
+
progress.update(1)
|
| 592 |
+
if ready:
|
| 593 |
+
current_frontier = task_index + 1
|
| 594 |
+
continue
|
| 595 |
+
if cache_path.exists():
|
| 596 |
+
remove_path(cache_path)
|
| 597 |
+
progress.close()
|
| 598 |
+
return current_frontier
|
| 599 |
+
progress.close()
|
| 600 |
+
return current_frontier
|
| 601 |
+
|
| 602 |
+
|
| 603 |
+
def acquire_lock(lock_path: Path) -> int | None:
|
| 604 |
+
lock_payload = {
|
| 605 |
+
"pid": os.getpid(),
|
| 606 |
+
"host": socket.gethostname(),
|
| 607 |
+
"time": time.time(),
|
| 608 |
+
}
|
| 609 |
+
while True:
|
| 610 |
+
try:
|
| 611 |
+
fd = os.open(str(lock_path), os.O_CREAT | os.O_EXCL | os.O_WRONLY)
|
| 612 |
+
os.write(fd, json.dumps(lock_payload).encode("utf-8"))
|
| 613 |
+
return fd
|
| 614 |
+
except FileExistsError:
|
| 615 |
+
try:
|
| 616 |
+
payload = json.loads(lock_path.read_text(encoding="utf-8"))
|
| 617 |
+
pid = int(payload.get("pid") or -1)
|
| 618 |
+
except Exception:
|
| 619 |
+
pid = -1
|
| 620 |
+
if pid <= 0 or not Path(f"/proc/{pid}").exists():
|
| 621 |
+
remove_path(lock_path)
|
| 622 |
+
continue
|
| 623 |
+
return None
|
| 624 |
+
|
| 625 |
+
|
| 626 |
+
def release_lock(fd: int | None, lock_path: Path) -> None:
|
| 627 |
+
if fd is not None:
|
| 628 |
+
try:
|
| 629 |
+
os.close(fd)
|
| 630 |
+
except OSError:
|
| 631 |
+
pass
|
| 632 |
+
remove_path(lock_path)
|
| 633 |
+
|
| 634 |
+
|
| 635 |
+
def init_worker(model_path: str) -> None:
|
| 636 |
+
global _WORKER_PROCESSOR, _WORKER_MODEL_PATH
|
| 637 |
+
_WORKER_MODEL_PATH = model_path
|
| 638 |
+
_WORKER_PROCESSOR = Qwen2_5OmniProcessor.from_pretrained(model_path, use_fast=False)
|
| 639 |
+
|
| 640 |
+
|
| 641 |
+
def get_worker_processor() -> Qwen2_5OmniProcessor:
|
| 642 |
+
global _WORKER_PROCESSOR
|
| 643 |
+
if _WORKER_PROCESSOR is None:
|
| 644 |
+
if not _WORKER_MODEL_PATH:
|
| 645 |
+
raise RuntimeError("Worker processor is not initialized")
|
| 646 |
+
_WORKER_PROCESSOR = Qwen2_5OmniProcessor.from_pretrained(_WORKER_MODEL_PATH, use_fast=False)
|
| 647 |
+
return _WORKER_PROCESSOR
|
| 648 |
+
|
| 649 |
+
|
| 650 |
+
def cache_one_video(task: dict) -> dict:
|
| 651 |
+
video_path = Path(task["video_path"])
|
| 652 |
+
decode_video_path = Path(task.get("decode_video_path") or video_path)
|
| 653 |
+
embed_root = Path(task["embed_root"])
|
| 654 |
+
cache_path = video_embed_cache_path(
|
| 655 |
+
video_path,
|
| 656 |
+
embed_root,
|
| 657 |
+
video_sample_fps=float(task["video_sample_fps"]),
|
| 658 |
+
video_size_pixels=int(task["video_size_pixels"]),
|
| 659 |
+
)
|
| 660 |
+
overwrite = bool(task["overwrite"])
|
| 661 |
+
existing_cache_check = str(task.get("existing_cache_check", "strict"))
|
| 662 |
+
task_index = int(task.get("task_index", -1))
|
| 663 |
+
result_base = {"path": str(cache_path), "task_index": task_index}
|
| 664 |
+
|
| 665 |
+
if cache_path.exists() and not overwrite:
|
| 666 |
+
if is_existing_cache_ready(cache_path, existing_cache_check):
|
| 667 |
+
return {"status": "skipped", **result_base}
|
| 668 |
+
remove_path(cache_path)
|
| 669 |
+
|
| 670 |
+
lock_path = cache_path.with_name(f"{cache_path.name}.lock")
|
| 671 |
+
lock_fd = acquire_lock(lock_path)
|
| 672 |
+
if lock_fd is None:
|
| 673 |
+
if cache_path.exists() and is_existing_cache_ready(cache_path, existing_cache_check):
|
| 674 |
+
return {"status": "skipped", **result_base}
|
| 675 |
+
return {"status": "locked", **result_base}
|
| 676 |
+
|
| 677 |
+
tmp_path = cache_path.with_name(f".{cache_path.name}.{os.getpid()}.{uuid.uuid4().hex}.tmp")
|
| 678 |
+
try:
|
| 679 |
+
if cache_path.exists() and not overwrite:
|
| 680 |
+
if is_existing_cache_ready(cache_path, existing_cache_check):
|
| 681 |
+
return {"status": "skipped", **result_base}
|
| 682 |
+
remove_path(cache_path)
|
| 683 |
+
|
| 684 |
+
if bool(task.get("direct_source_clip", False)):
|
| 685 |
+
video, source_video_fps, video_duration_sec = load_video_clip_array(
|
| 686 |
+
decode_video_path,
|
| 687 |
+
start_sec=float(task["clip_start_sec"]),
|
| 688 |
+
duration_sec=float(task["clip_duration_sec"]),
|
| 689 |
+
video_sample_fps=float(task["video_sample_fps"]),
|
| 690 |
+
video_size_pixels=int(task["video_size_pixels"]),
|
| 691 |
+
video_decoder=str(task["video_decoder"]),
|
| 692 |
+
)
|
| 693 |
+
else:
|
| 694 |
+
video, source_video_fps, video_duration_sec = load_video_array(
|
| 695 |
+
decode_video_path,
|
| 696 |
+
video_sample_fps=float(task["video_sample_fps"]),
|
| 697 |
+
video_size_pixels=int(task["video_size_pixels"]),
|
| 698 |
+
video_decoder=str(task["video_decoder"]),
|
| 699 |
+
)
|
| 700 |
+
processor = get_worker_processor()
|
| 701 |
+
video_inputs = processor.video_processor(
|
| 702 |
+
videos=[video],
|
| 703 |
+
size={
|
| 704 |
+
"shortest_edge": int(task["video_size_pixels"]),
|
| 705 |
+
"longest_edge": int(task["video_size_pixels"]),
|
| 706 |
+
},
|
| 707 |
+
)
|
| 708 |
+
video_second_per_grid = np.asarray(
|
| 709 |
+
[float(processor.video_processor.temporal_patch_size) / float(task["video_sample_fps"])],
|
| 710 |
+
dtype=np.float32,
|
| 711 |
+
)
|
| 712 |
+
dtype = np.float16 if str(task["dtype"]) == "float16" else np.float32
|
| 713 |
+
payload = {
|
| 714 |
+
"pixel_values_videos": np.asarray(video_inputs["pixel_values_videos"], dtype=dtype),
|
| 715 |
+
"video_grid_thw": np.asarray(video_inputs["video_grid_thw"], dtype=np.int64),
|
| 716 |
+
"video_second_per_grid": video_second_per_grid,
|
| 717 |
+
"video_sample_fps": np.asarray([float(task["video_sample_fps"])], dtype=np.float32),
|
| 718 |
+
"video_size_pixels": np.asarray([int(task["video_size_pixels"])], dtype=np.int64),
|
| 719 |
+
"source_video_fps": np.asarray([float(source_video_fps)], dtype=np.float32),
|
| 720 |
+
"video_duration_sec": np.asarray([float(video_duration_sec)], dtype=np.float32),
|
| 721 |
+
}
|
| 722 |
+
with tmp_path.open("wb") as f:
|
| 723 |
+
if bool(task["save_compressed"]):
|
| 724 |
+
np.savez_compressed(f, **payload)
|
| 725 |
+
else:
|
| 726 |
+
np.savez(f, **payload)
|
| 727 |
+
if not is_valid_cache(tmp_path):
|
| 728 |
+
raise RuntimeError(f"Invalid cache was written: {tmp_path}")
|
| 729 |
+
tmp_path.replace(cache_path)
|
| 730 |
+
return {"status": "cached", **result_base}
|
| 731 |
+
except Exception as exc:
|
| 732 |
+
remove_path(tmp_path)
|
| 733 |
+
return {"status": "failed", "error": str(exc), **result_base}
|
| 734 |
+
finally:
|
| 735 |
+
release_lock(lock_fd, lock_path)
|
| 736 |
+
|
| 737 |
+
|
| 738 |
+
def build_cache_task(args: argparse.Namespace, video_task: dict, embed_root: Path) -> dict:
|
| 739 |
+
task = {
|
| 740 |
+
"video_path": str(video_task["video_path"]),
|
| 741 |
+
"decode_video_path": str(video_task.get("decode_video_path") or video_task["video_path"]),
|
| 742 |
+
"embed_root": str(embed_root),
|
| 743 |
+
"video_sample_fps": float(args.video_sample_fps),
|
| 744 |
+
"video_size_pixels": int(args.video_size_pixels),
|
| 745 |
+
"video_decoder": str(args.video_decoder),
|
| 746 |
+
"dtype": str(args.dtype),
|
| 747 |
+
"save_compressed": bool(args.save_compressed),
|
| 748 |
+
"overwrite": bool(args.overwrite),
|
| 749 |
+
"existing_cache_check": str(args.existing_cache_check),
|
| 750 |
+
}
|
| 751 |
+
if "clip_start_sec" in video_task:
|
| 752 |
+
task.update(
|
| 753 |
+
{
|
| 754 |
+
"direct_source_clip": True,
|
| 755 |
+
"clip_start_sec": float(video_task["clip_start_sec"]),
|
| 756 |
+
"clip_duration_sec": float(video_task["clip_duration_sec"]),
|
| 757 |
+
}
|
| 758 |
+
)
|
| 759 |
+
if "task_index" in video_task:
|
| 760 |
+
task["task_index"] = int(video_task["task_index"])
|
| 761 |
+
return task
|
| 762 |
+
|
| 763 |
+
|
| 764 |
+
def update_result_counters(
|
| 765 |
+
result: dict,
|
| 766 |
+
tracker: ResumeTracker,
|
| 767 |
+
counters: dict[str, int],
|
| 768 |
+
) -> None:
|
| 769 |
+
status = result["status"]
|
| 770 |
+
task_index = result.get("task_index")
|
| 771 |
+
if status == "cached":
|
| 772 |
+
counters["cached"] += 1
|
| 773 |
+
tracker.mark_complete(task_index)
|
| 774 |
+
elif status == "skipped":
|
| 775 |
+
counters["skipped"] += 1
|
| 776 |
+
tracker.mark_complete(task_index)
|
| 777 |
+
elif status == "locked":
|
| 778 |
+
counters["locked"] += 1
|
| 779 |
+
else:
|
| 780 |
+
counters["failed"] += 1
|
| 781 |
+
print(f"[FAILED] {result['path']}: {result.get('error', '')}")
|
| 782 |
+
tracker.flush()
|
| 783 |
+
|
| 784 |
+
|
| 785 |
+
def run_fast_resume_cache(args: argparse.Namespace, embed_root: Path) -> None:
|
| 786 |
+
frontier, known_total_count, state_path = load_resume_frontier(args, embed_root)
|
| 787 |
+
if known_total_count is None:
|
| 788 |
+
known_total_count = load_known_total_count(args, embed_root)
|
| 789 |
+
if known_total_count is None:
|
| 790 |
+
known_total_count = count_metadata_rows(args)
|
| 791 |
+
write_manifest(args, embed_root, known_total_count)
|
| 792 |
+
extended_frontier = extend_resume_frontier(args, embed_root, frontier)
|
| 793 |
+
tracker = ResumeTracker(
|
| 794 |
+
path=state_path,
|
| 795 |
+
args=args,
|
| 796 |
+
embed_root=embed_root,
|
| 797 |
+
frontier=extended_frontier,
|
| 798 |
+
total_count=known_total_count,
|
| 799 |
+
)
|
| 800 |
+
tracker.flush(force=True)
|
| 801 |
+
|
| 802 |
+
if known_total_count is not None:
|
| 803 |
+
print(
|
| 804 |
+
f"[SHARD] cache shard={args.shard_id}/{args.shard_num} mode=pre-sharded "
|
| 805 |
+
f"resume={extended_frontier}/{known_total_count}"
|
| 806 |
+
)
|
| 807 |
+
write_manifest(args, embed_root, known_total_count)
|
| 808 |
+
else:
|
| 809 |
+
print(
|
| 810 |
+
f"[SHARD] cache shard={args.shard_id}/{args.shard_num} mode=pre-sharded "
|
| 811 |
+
f"resume={extended_frontier}/?"
|
| 812 |
+
)
|
| 813 |
+
|
| 814 |
+
counters = {
|
| 815 |
+
"cached": 0,
|
| 816 |
+
"skipped": 0,
|
| 817 |
+
"locked": 0,
|
| 818 |
+
"failed": 0,
|
| 819 |
+
"invalid": 0,
|
| 820 |
+
}
|
| 821 |
+
total_hint = None
|
| 822 |
+
if tracker.total_count is not None:
|
| 823 |
+
total_hint = max(0, int(tracker.total_count) - int(extended_frontier))
|
| 824 |
+
|
| 825 |
+
if args.num_workers == 1:
|
| 826 |
+
progress = tqdm(total=total_hint, desc="cache omni video")
|
| 827 |
+
total_video_paths = extended_frontier
|
| 828 |
+
for video_task in iter_metadata_tasks_stream(args, start_index=extended_frontier):
|
| 829 |
+
total_video_paths = int(video_task["task_index"]) + 1
|
| 830 |
+
task = build_cache_task(args, video_task, embed_root)
|
| 831 |
+
result = cache_one_video(task)
|
| 832 |
+
update_result_counters(result, tracker, counters)
|
| 833 |
+
progress.update(1)
|
| 834 |
+
progress.close()
|
| 835 |
+
else:
|
| 836 |
+
max_in_flight = max(int(args.num_workers) * 4, int(args.num_workers))
|
| 837 |
+
progress = tqdm(total=total_hint, desc=f"cache omni video x{args.num_workers}")
|
| 838 |
+
total_video_paths = extended_frontier
|
| 839 |
+
with ProcessPoolExecutor(
|
| 840 |
+
max_workers=args.num_workers,
|
| 841 |
+
initializer=init_worker,
|
| 842 |
+
initargs=(args.model_path,),
|
| 843 |
+
) as executor:
|
| 844 |
+
pending_futures: dict = {}
|
| 845 |
+
task_iterator = iter_metadata_tasks_stream(args, start_index=extended_frontier)
|
| 846 |
+
exhausted = False
|
| 847 |
+
while True:
|
| 848 |
+
while not exhausted and len(pending_futures) < max_in_flight:
|
| 849 |
+
try:
|
| 850 |
+
video_task = next(task_iterator)
|
| 851 |
+
except StopIteration:
|
| 852 |
+
exhausted = True
|
| 853 |
+
break
|
| 854 |
+
total_video_paths = int(video_task["task_index"]) + 1
|
| 855 |
+
task = build_cache_task(args, video_task, embed_root)
|
| 856 |
+
future = executor.submit(cache_one_video, task)
|
| 857 |
+
pending_futures[future] = None
|
| 858 |
+
|
| 859 |
+
if not pending_futures:
|
| 860 |
+
break
|
| 861 |
+
|
| 862 |
+
done, _ = wait(tuple(pending_futures.keys()), return_when=FIRST_COMPLETED)
|
| 863 |
+
for future in done:
|
| 864 |
+
pending_futures.pop(future, None)
|
| 865 |
+
result = future.result()
|
| 866 |
+
update_result_counters(result, tracker, counters)
|
| 867 |
+
progress.update(1)
|
| 868 |
+
progress.close()
|
| 869 |
+
|
| 870 |
+
if tracker.total_count is None:
|
| 871 |
+
tracker.maybe_set_total_count(total_video_paths)
|
| 872 |
+
else:
|
| 873 |
+
tracker.maybe_set_total_count(max(int(tracker.total_count), int(total_video_paths)))
|
| 874 |
+
tracker.flush(force=True)
|
| 875 |
+
if total_video_paths > 0:
|
| 876 |
+
write_manifest(args, embed_root, total_video_paths if tracker.total_count is None else int(tracker.total_count))
|
| 877 |
+
|
| 878 |
+
print(
|
| 879 |
+
f"cached={counters['cached']} skipped={counters['skipped']} invalid_removed={counters['invalid']} "
|
| 880 |
+
f"locked={counters['locked']} failed={counters['failed']} embed_root={embed_root}"
|
| 881 |
+
)
|
| 882 |
+
if counters["failed"] > 0:
|
| 883 |
+
raise SystemExit(1)
|
| 884 |
+
if counters["locked"] > 0:
|
| 885 |
+
raise SystemExit(2)
|
| 886 |
+
|
| 887 |
+
|
| 888 |
+
def main() -> None:
|
| 889 |
+
args = parse_args()
|
| 890 |
+
if args.num_workers <= 0:
|
| 891 |
+
raise ValueError("--num-workers must be positive")
|
| 892 |
+
if args.shard_num <= 0:
|
| 893 |
+
raise ValueError("--shard-num must be positive")
|
| 894 |
+
if args.shard_id < 0 or args.shard_id >= args.shard_num:
|
| 895 |
+
raise ValueError("--shard-id must satisfy 0 <= shard_id < shard_num")
|
| 896 |
+
|
| 897 |
+
embed_root = Path(args.embed_root)
|
| 898 |
+
embed_root.mkdir(parents=True, exist_ok=True)
|
| 899 |
+
|
| 900 |
+
if fast_resume_enabled(args):
|
| 901 |
+
run_fast_resume_cache(args, embed_root)
|
| 902 |
+
return
|
| 903 |
+
|
| 904 |
+
video_tasks = iter_video_tasks(args)
|
| 905 |
+
if args.limit > 0:
|
| 906 |
+
video_tasks = video_tasks[: args.limit]
|
| 907 |
+
total_video_paths = len(video_tasks)
|
| 908 |
+
if args.shard_num > 1 and not args.input_already_sharded:
|
| 909 |
+
video_tasks = [
|
| 910 |
+
video_task for video_task in video_tasks
|
| 911 |
+
if stable_hash_mod(str(video_task["video_path"]), args.shard_num) == args.shard_id
|
| 912 |
+
]
|
| 913 |
+
if args.shard_num > 1:
|
| 914 |
+
mode = "pre-sharded" if args.input_already_sharded else "hash"
|
| 915 |
+
print(
|
| 916 |
+
f"[SHARD] cache shard={args.shard_id}/{args.shard_num} mode={mode} "
|
| 917 |
+
f"videos={len(video_tasks)}/{total_video_paths}"
|
| 918 |
+
)
|
| 919 |
+
|
| 920 |
+
write_manifest(args, embed_root, total_video_paths)
|
| 921 |
+
|
| 922 |
+
pending_tasks = []
|
| 923 |
+
cached = 0
|
| 924 |
+
skipped = 0
|
| 925 |
+
locked = 0
|
| 926 |
+
failed = 0
|
| 927 |
+
invalid = 0
|
| 928 |
+
scan_progress = tqdm(video_tasks, total=len(video_tasks), desc="scan cache state")
|
| 929 |
+
for video_task in scan_progress:
|
| 930 |
+
video_path = Path(video_task["video_path"])
|
| 931 |
+
cache_path = video_embed_cache_path(
|
| 932 |
+
video_path,
|
| 933 |
+
embed_root,
|
| 934 |
+
video_sample_fps=args.video_sample_fps,
|
| 935 |
+
video_size_pixels=args.video_size_pixels,
|
| 936 |
+
)
|
| 937 |
+
if cache_path.exists() and not args.overwrite:
|
| 938 |
+
if is_existing_cache_ready(cache_path, args.existing_cache_check):
|
| 939 |
+
skipped += 1
|
| 940 |
+
continue
|
| 941 |
+
remove_path(cache_path)
|
| 942 |
+
invalid += 1
|
| 943 |
+
pending_tasks.append(build_cache_task(args, video_task, embed_root))
|
| 944 |
+
|
| 945 |
+
if pending_tasks:
|
| 946 |
+
if args.num_workers == 1:
|
| 947 |
+
init_worker(args.model_path)
|
| 948 |
+
iterator = (cache_one_video(task) for task in pending_tasks)
|
| 949 |
+
progress = tqdm(iterator, total=len(pending_tasks), desc="cache omni video")
|
| 950 |
+
for result in progress:
|
| 951 |
+
status = result["status"]
|
| 952 |
+
if status == "cached":
|
| 953 |
+
cached += 1
|
| 954 |
+
elif status == "skipped":
|
| 955 |
+
skipped += 1
|
| 956 |
+
elif status == "locked":
|
| 957 |
+
locked += 1
|
| 958 |
+
else:
|
| 959 |
+
failed += 1
|
| 960 |
+
print(f"[FAILED] {result['path']}: {result.get('error', '')}")
|
| 961 |
+
else:
|
| 962 |
+
with ProcessPoolExecutor(
|
| 963 |
+
max_workers=args.num_workers,
|
| 964 |
+
initializer=init_worker,
|
| 965 |
+
initargs=(args.model_path,),
|
| 966 |
+
) as executor:
|
| 967 |
+
futures = [executor.submit(cache_one_video, task) for task in pending_tasks]
|
| 968 |
+
for future in tqdm(
|
| 969 |
+
as_completed(futures),
|
| 970 |
+
total=len(futures),
|
| 971 |
+
desc=f"cache omni video x{args.num_workers}",
|
| 972 |
+
):
|
| 973 |
+
result = future.result()
|
| 974 |
+
status = result["status"]
|
| 975 |
+
if status == "cached":
|
| 976 |
+
cached += 1
|
| 977 |
+
elif status == "skipped":
|
| 978 |
+
skipped += 1
|
| 979 |
+
elif status == "locked":
|
| 980 |
+
locked += 1
|
| 981 |
+
else:
|
| 982 |
+
failed += 1
|
| 983 |
+
print(f"[FAILED] {result['path']}: {result.get('error', '')}")
|
| 984 |
+
|
| 985 |
+
print(
|
| 986 |
+
f"cached={cached} skipped={skipped} invalid_removed={invalid} "
|
| 987 |
+
f"locked={locked} failed={failed} embed_root={embed_root}"
|
| 988 |
+
)
|
| 989 |
+
if failed > 0:
|
| 990 |
+
raise SystemExit(1)
|
| 991 |
+
if locked > 0:
|
| 992 |
+
raise SystemExit(2)
|
| 993 |
+
|
| 994 |
+
|
| 995 |
+
if __name__ == "__main__":
|
| 996 |
+
main()
|
cache_video_omni_embeds.sh
ADDED
|
@@ -0,0 +1,275 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#!/usr/bin/env bash
|
| 2 |
+
set -euo pipefail
|
| 3 |
+
|
| 4 |
+
cd /cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/Psi0-yth-v18
|
| 5 |
+
PSI_PYTHON_BIN=${PSI_PYTHON_BIN:-}
|
| 6 |
+
PSI_PYTHON_ENV=${PSI_PYTHON_ENV:-}
|
| 7 |
+
if [ -z "$PSI_PYTHON_BIN" ] && [ -n "$PSI_PYTHON_ENV" ]; then
|
| 8 |
+
if [ ! -x "$PSI_PYTHON_ENV/bin/python" ]; then
|
| 9 |
+
echo "ERROR: PSI_PYTHON_ENV does not contain an executable python: $PSI_PYTHON_ENV/bin/python" >&2
|
| 10 |
+
exit 2
|
| 11 |
+
fi
|
| 12 |
+
source "$PSI_PYTHON_ENV/bin/activate"
|
| 13 |
+
fi
|
| 14 |
+
[ -f .env ] && source .env
|
| 15 |
+
|
| 16 |
+
export PYTHONPATH=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/Psi0-yth-v18/src
|
| 17 |
+
export CUDA_VISIBLE_DEVICES=${CUDA_VISIBLE_DEVICES:-0}
|
| 18 |
+
export OMP_NUM_THREADS=${OMP_NUM_THREADS:-1}
|
| 19 |
+
PYTHON_BIN=${PSI_PYTHON_BIN:-$(command -v python)}
|
| 20 |
+
if [ ! -x "$PYTHON_BIN" ]; then
|
| 21 |
+
echo "ERROR: python executable is not valid: $PYTHON_BIN" >&2
|
| 22 |
+
exit 2
|
| 23 |
+
fi
|
| 24 |
+
|
| 25 |
+
DATASET_ROOT=${PSI_VIDEO_CLIP_DATASET_ROOT:-/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/videogen/psi0_video_clip_50f_rand8}
|
| 26 |
+
DANCE_CLIP_ROOT=$DATASET_ROOT/dance_1723_video_10s
|
| 27 |
+
MOTION_EXAMPLE_CLIP_ROOT=$DATASET_ROOT/motion_example_0506_demo
|
| 28 |
+
DANCE_CLIP_EMBED_ROOT=${PSI_DANCE_VIDEO_CLIP_EMBED_ROOT:-$DANCE_CLIP_ROOT/embeds}
|
| 29 |
+
MOTION_EXAMPLE_CLIP_EMBED_ROOT=${PSI_MOTION_EXAMPLE_VIDEO_CLIP_EMBED_ROOT:-$MOTION_EXAMPLE_CLIP_ROOT/embeds}
|
| 30 |
+
|
| 31 |
+
DANCE_MOTION_ROOT=${PSI_DANCE_MOTION_ROOT:-/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/GR00T-WholeBodyControl/gear_sonic_deploy/reference/converted/psi0_dance/gvhmr}
|
| 32 |
+
DANCE_VIDEO_ROOT=${PSI_DANCE_VIDEO_ROOT:-/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/dance_1723_video_psi0/dance_1723_video_10s}
|
| 33 |
+
MOTION_EXAMPLE_ROOT=${PSI_VIDEO_DEMO_ROOT:-/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/videogen/motion_example_0506_demo}
|
| 34 |
+
|
| 35 |
+
MODEL_PATH=${PSI_PRETRAIN_BACKBONE_PATH:-/cpfs01/VideoGen/public_ckpt/Qwen/Qwen2.5-Omni-3B}
|
| 36 |
+
VIDEO_SAMPLE_FPS=${VIDEO_SAMPLE_FPS:-32}
|
| 37 |
+
VIDEO_SIZE_PIXELS=${VIDEO_SIZE_PIXELS:-50176}
|
| 38 |
+
VIDEO_DECODER=${VIDEO_DECODER:-ffmpeg}
|
| 39 |
+
CLIP_FRAMES=${CLIP_FRAMES:-50}
|
| 40 |
+
ACTION_CHUNK_FRAMES=${ACTION_CHUNK_FRAMES:-30}
|
| 41 |
+
CHUNK_STRIDE_FRAMES=${CHUNK_STRIDE_FRAMES:-20}
|
| 42 |
+
MOTION_FPS=${MOTION_FPS:-50}
|
| 43 |
+
CLIPS_PER_VIDEO=${CLIPS_PER_VIDEO:-2}
|
| 44 |
+
CLIP_SELECTION_MODE=${CLIP_SELECTION_MODE:-middle_random}
|
| 45 |
+
CLIP_RANDOM_SEED=${CLIP_RANDOM_SEED:-42}
|
| 46 |
+
CLIP_MIDDLE_FRACTION=${CLIP_MIDDLE_FRACTION:-0.8}
|
| 47 |
+
CHUNK_PROGRESS_INTERVAL=${CHUNK_PROGRESS_INTERVAL:-0}
|
| 48 |
+
BUILD_NUM_WORKERS=${BUILD_NUM_WORKERS:-32}
|
| 49 |
+
CHUNK_NUM_WORKERS=${CHUNK_NUM_WORKERS:-1}
|
| 50 |
+
FFMPEG_THREADS=${FFMPEG_THREADS:-1}
|
| 51 |
+
X264_PRESET=${X264_PRESET:-}
|
| 52 |
+
EXISTING_VIDEO_CHECK=${EXISTING_VIDEO_CHECK:-ffprobe}
|
| 53 |
+
EMBED_NUM_WORKERS=${EMBED_NUM_WORKERS:-64}
|
| 54 |
+
EMBED_SAVE_COMPRESSED=${EMBED_SAVE_COMPRESSED:-0}
|
| 55 |
+
EXISTING_CACHE_CHECK=${EXISTING_CACHE_CHECK:-strict}
|
| 56 |
+
BUILD_CLIPS=${BUILD_CLIPS:-1}
|
| 57 |
+
CACHE_EMBEDS=${CACHE_EMBEDS:-1}
|
| 58 |
+
BUILD_DANCE=${BUILD_DANCE:-1}
|
| 59 |
+
BUILD_MOTION_EXAMPLE=${BUILD_MOTION_EXAMPLE:-1}
|
| 60 |
+
CACHE_DANCE=${CACHE_DANCE:-1}
|
| 61 |
+
CACHE_MOTION_EXAMPLE=${CACHE_MOTION_EXAMPLE:-1}
|
| 62 |
+
LIMIT=${LIMIT:-0}
|
| 63 |
+
OVERWRITE_CLIPS=${OVERWRITE_CLIPS:-0}
|
| 64 |
+
OVERWRITE_EMBEDS=${OVERWRITE_EMBEDS:-0}
|
| 65 |
+
SHARD_NUM=${SHARD_NUM:-1}
|
| 66 |
+
SHARD_ID=${SHARD_ID:-0}
|
| 67 |
+
METADATA_ONLY=${METADATA_ONLY:-0}
|
| 68 |
+
DIRECT_SOURCE_CLIPS=${DIRECT_SOURCE_CLIPS:-0}
|
| 69 |
+
|
| 70 |
+
if ! [[ "$SHARD_NUM" =~ ^[0-9]+$ ]] || ! [[ "$SHARD_ID" =~ ^[0-9]+$ ]]; then
|
| 71 |
+
echo "SHARD_NUM and SHARD_ID must be non-negative integers" >&2
|
| 72 |
+
exit 2
|
| 73 |
+
fi
|
| 74 |
+
if (( SHARD_NUM <= 0 )); then
|
| 75 |
+
echo "SHARD_NUM must be positive" >&2
|
| 76 |
+
exit 2
|
| 77 |
+
fi
|
| 78 |
+
if (( SHARD_ID >= SHARD_NUM )); then
|
| 79 |
+
echo "SHARD_ID must satisfy 0 <= SHARD_ID < SHARD_NUM" >&2
|
| 80 |
+
exit 2
|
| 81 |
+
fi
|
| 82 |
+
printf -v SHARD_ID_PAD "%05d" "$SHARD_ID"
|
| 83 |
+
printf -v SHARD_NUM_PAD "%05d" "$SHARD_NUM"
|
| 84 |
+
|
| 85 |
+
clip_overwrite_args=()
|
| 86 |
+
if [ "$OVERWRITE_CLIPS" = "1" ]; then
|
| 87 |
+
clip_overwrite_args+=(--overwrite)
|
| 88 |
+
fi
|
| 89 |
+
|
| 90 |
+
embed_overwrite_args=()
|
| 91 |
+
if [ "$OVERWRITE_EMBEDS" = "1" ]; then
|
| 92 |
+
embed_overwrite_args+=(--overwrite)
|
| 93 |
+
fi
|
| 94 |
+
|
| 95 |
+
embed_compress_args=()
|
| 96 |
+
if [ "$EMBED_SAVE_COMPRESSED" = "1" ]; then
|
| 97 |
+
embed_compress_args+=(--save-compressed)
|
| 98 |
+
fi
|
| 99 |
+
|
| 100 |
+
limit_args=()
|
| 101 |
+
if [ "$LIMIT" -gt 0 ]; then
|
| 102 |
+
limit_args+=(--limit "$LIMIT")
|
| 103 |
+
fi
|
| 104 |
+
|
| 105 |
+
shard_args=()
|
| 106 |
+
if (( SHARD_NUM > 1 )); then
|
| 107 |
+
shard_args+=(--shard-num "$SHARD_NUM" --shard-id "$SHARD_ID")
|
| 108 |
+
fi
|
| 109 |
+
|
| 110 |
+
chunk_progress_args=()
|
| 111 |
+
if [ "$CHUNK_PROGRESS_INTERVAL" -gt 0 ]; then
|
| 112 |
+
chunk_progress_args+=(--chunk-progress-interval "$CHUNK_PROGRESS_INTERVAL")
|
| 113 |
+
fi
|
| 114 |
+
|
| 115 |
+
metadata_only_args=()
|
| 116 |
+
if [ "$METADATA_ONLY" = "1" ]; then
|
| 117 |
+
metadata_only_args+=(--metadata-only)
|
| 118 |
+
fi
|
| 119 |
+
|
| 120 |
+
direct_source_clip_args=()
|
| 121 |
+
if [ "$DIRECT_SOURCE_CLIPS" = "1" ]; then
|
| 122 |
+
direct_source_clip_args+=(--direct-source-clips)
|
| 123 |
+
fi
|
| 124 |
+
|
| 125 |
+
cache_metadata_jsonl() {
|
| 126 |
+
local clip_root="$1"
|
| 127 |
+
local shard_metadata="metadata.shard-${SHARD_ID_PAD}-of-${SHARD_NUM_PAD}.jsonl"
|
| 128 |
+
if (( SHARD_NUM > 1 )) && [ -f "$clip_root/$shard_metadata" ]; then
|
| 129 |
+
printf "%s" "$shard_metadata"
|
| 130 |
+
else
|
| 131 |
+
printf "%s" "metadata.jsonl"
|
| 132 |
+
fi
|
| 133 |
+
}
|
| 134 |
+
|
| 135 |
+
echo "Dataset root: $DATASET_ROOT"
|
| 136 |
+
echo "Dance clip embed root: $DANCE_CLIP_EMBED_ROOT"
|
| 137 |
+
echo "Motion example clip embed root: $MOTION_EXAMPLE_CLIP_EMBED_ROOT"
|
| 138 |
+
echo "Build clips: $BUILD_CLIPS"
|
| 139 |
+
echo "Cache embeds: $CACHE_EMBEDS"
|
| 140 |
+
echo "Build dance: $BUILD_DANCE"
|
| 141 |
+
echo "Build motion_example: $BUILD_MOTION_EXAMPLE"
|
| 142 |
+
echo "Cache dance: $CACHE_DANCE"
|
| 143 |
+
echo "Cache motion_example: $CACHE_MOTION_EXAMPLE"
|
| 144 |
+
echo "Clip frames: $CLIP_FRAMES"
|
| 145 |
+
echo "Action chunk frames: $ACTION_CHUNK_FRAMES"
|
| 146 |
+
echo "Chunk stride frames: $CHUNK_STRIDE_FRAMES"
|
| 147 |
+
echo "Clips per video: $CLIPS_PER_VIDEO"
|
| 148 |
+
echo "Clip selection mode: $CLIP_SELECTION_MODE"
|
| 149 |
+
echo "Clip random seed: $CLIP_RANDOM_SEED"
|
| 150 |
+
echo "Clip middle fraction: $CLIP_MIDDLE_FRACTION"
|
| 151 |
+
echo "Chunk progress interval: $CHUNK_PROGRESS_INTERVAL"
|
| 152 |
+
echo "Build workers: $BUILD_NUM_WORKERS"
|
| 153 |
+
echo "Chunk workers per source: $CHUNK_NUM_WORKERS"
|
| 154 |
+
echo "FFmpeg threads: $FFMPEG_THREADS"
|
| 155 |
+
echo "x264 preset: ${X264_PRESET:-<default>}"
|
| 156 |
+
echo "Existing video check: $EXISTING_VIDEO_CHECK"
|
| 157 |
+
echo "Embed workers: $EMBED_NUM_WORKERS"
|
| 158 |
+
echo "Embed compressed: $EMBED_SAVE_COMPRESSED"
|
| 159 |
+
echo "Existing cache check: $EXISTING_CACHE_CHECK"
|
| 160 |
+
echo "Video sample fps: $VIDEO_SAMPLE_FPS"
|
| 161 |
+
echo "Video size pixels: $VIDEO_SIZE_PIXELS"
|
| 162 |
+
echo "Shard: $SHARD_ID/$SHARD_NUM"
|
| 163 |
+
echo "Metadata only: $METADATA_ONLY"
|
| 164 |
+
echo "Direct source clips: $DIRECT_SOURCE_CLIPS"
|
| 165 |
+
echo "Python: $PYTHON_BIN"
|
| 166 |
+
|
| 167 |
+
if [ "$CACHE_EMBEDS" = "1" ]; then
|
| 168 |
+
"$PYTHON_BIN" - <<'PY'
|
| 169 |
+
import sys
|
| 170 |
+
from transformers import Qwen2_5OmniProcessor
|
| 171 |
+
print("Python executable:", sys.executable)
|
| 172 |
+
print("Qwen2_5OmniProcessor import: ok")
|
| 173 |
+
PY
|
| 174 |
+
fi
|
| 175 |
+
|
| 176 |
+
if [ "$BUILD_CLIPS" = "1" ] && [ "$BUILD_DANCE" = "1" ]; then
|
| 177 |
+
"$PYTHON_BIN" scripts/data/build_video_clip_dataset.py \
|
| 178 |
+
--dataset dance \
|
| 179 |
+
--output-root "$DANCE_CLIP_ROOT" \
|
| 180 |
+
--dance-motion-root "$DANCE_MOTION_ROOT" \
|
| 181 |
+
--dance-video-root "$DANCE_VIDEO_ROOT" \
|
| 182 |
+
--clip-frames "$CLIP_FRAMES" \
|
| 183 |
+
--action-chunk-frames "$ACTION_CHUNK_FRAMES" \
|
| 184 |
+
--chunk-stride-frames "$CHUNK_STRIDE_FRAMES" \
|
| 185 |
+
--motion-fps "$MOTION_FPS" \
|
| 186 |
+
--clips-per-video "$CLIPS_PER_VIDEO" \
|
| 187 |
+
--clip-selection-mode "$CLIP_SELECTION_MODE" \
|
| 188 |
+
--random-seed "$CLIP_RANDOM_SEED" \
|
| 189 |
+
--middle-fraction "$CLIP_MIDDLE_FRACTION" \
|
| 190 |
+
--num-workers "$BUILD_NUM_WORKERS" \
|
| 191 |
+
--chunk-workers "$CHUNK_NUM_WORKERS" \
|
| 192 |
+
--ffmpeg-threads "$FFMPEG_THREADS" \
|
| 193 |
+
--x264-preset "$X264_PRESET" \
|
| 194 |
+
--existing-video-check "$EXISTING_VIDEO_CHECK" \
|
| 195 |
+
"${shard_args[@]}" \
|
| 196 |
+
"${chunk_progress_args[@]}" \
|
| 197 |
+
"${limit_args[@]}" \
|
| 198 |
+
"${metadata_only_args[@]}" \
|
| 199 |
+
"${clip_overwrite_args[@]}"
|
| 200 |
+
fi
|
| 201 |
+
|
| 202 |
+
if [ "$BUILD_CLIPS" = "1" ] && [ "$BUILD_MOTION_EXAMPLE" = "1" ]; then
|
| 203 |
+
"$PYTHON_BIN" scripts/data/build_video_clip_dataset.py \
|
| 204 |
+
--dataset motion_example \
|
| 205 |
+
--output-root "$MOTION_EXAMPLE_CLIP_ROOT" \
|
| 206 |
+
--motion-example-root "$MOTION_EXAMPLE_ROOT" \
|
| 207 |
+
--clip-frames "$CLIP_FRAMES" \
|
| 208 |
+
--action-chunk-frames "$ACTION_CHUNK_FRAMES" \
|
| 209 |
+
--chunk-stride-frames "$CHUNK_STRIDE_FRAMES" \
|
| 210 |
+
--motion-fps "$MOTION_FPS" \
|
| 211 |
+
--clips-per-video "$CLIPS_PER_VIDEO" \
|
| 212 |
+
--clip-selection-mode "$CLIP_SELECTION_MODE" \
|
| 213 |
+
--random-seed "$CLIP_RANDOM_SEED" \
|
| 214 |
+
--middle-fraction "$CLIP_MIDDLE_FRACTION" \
|
| 215 |
+
--num-workers "$BUILD_NUM_WORKERS" \
|
| 216 |
+
--chunk-workers "$CHUNK_NUM_WORKERS" \
|
| 217 |
+
--ffmpeg-threads "$FFMPEG_THREADS" \
|
| 218 |
+
--x264-preset "$X264_PRESET" \
|
| 219 |
+
--existing-video-check "$EXISTING_VIDEO_CHECK" \
|
| 220 |
+
"${shard_args[@]}" \
|
| 221 |
+
"${chunk_progress_args[@]}" \
|
| 222 |
+
"${limit_args[@]}" \
|
| 223 |
+
"${metadata_only_args[@]}" \
|
| 224 |
+
"${clip_overwrite_args[@]}"
|
| 225 |
+
fi
|
| 226 |
+
|
| 227 |
+
if [ "$CACHE_EMBEDS" = "1" ] && [ "$CACHE_DANCE" = "1" ]; then
|
| 228 |
+
dance_metadata_jsonl=$(cache_metadata_jsonl "$DANCE_CLIP_ROOT")
|
| 229 |
+
dance_cache_shard_args=("${shard_args[@]}")
|
| 230 |
+
if (( SHARD_NUM > 1 )) && [ "$dance_metadata_jsonl" != "metadata.jsonl" ]; then
|
| 231 |
+
dance_cache_shard_args+=(--input-already-sharded)
|
| 232 |
+
fi
|
| 233 |
+
"$PYTHON_BIN" cache_video_omni_embeds.py \
|
| 234 |
+
--dataset-root "$DANCE_CLIP_ROOT" \
|
| 235 |
+
--metadata-jsonl "$dance_metadata_jsonl" \
|
| 236 |
+
--embed-root "$DANCE_CLIP_EMBED_ROOT" \
|
| 237 |
+
--model-path "$MODEL_PATH" \
|
| 238 |
+
--video-sample-fps "$VIDEO_SAMPLE_FPS" \
|
| 239 |
+
--video-size-pixels "$VIDEO_SIZE_PIXELS" \
|
| 240 |
+
--video-decoder "$VIDEO_DECODER" \
|
| 241 |
+
--dtype float16 \
|
| 242 |
+
--num-workers "$EMBED_NUM_WORKERS" \
|
| 243 |
+
--existing-cache-check "$EXISTING_CACHE_CHECK" \
|
| 244 |
+
"${dance_cache_shard_args[@]}" \
|
| 245 |
+
"${limit_args[@]}" \
|
| 246 |
+
"${direct_source_clip_args[@]}" \
|
| 247 |
+
"${embed_compress_args[@]}" \
|
| 248 |
+
"${embed_overwrite_args[@]}"
|
| 249 |
+
fi
|
| 250 |
+
|
| 251 |
+
if [ "$CACHE_EMBEDS" = "1" ] && [ "$CACHE_MOTION_EXAMPLE" = "1" ]; then
|
| 252 |
+
motion_metadata_jsonl=$(cache_metadata_jsonl "$MOTION_EXAMPLE_CLIP_ROOT")
|
| 253 |
+
motion_cache_shard_args=("${shard_args[@]}")
|
| 254 |
+
if (( SHARD_NUM > 1 )) && [ "$motion_metadata_jsonl" != "metadata.jsonl" ]; then
|
| 255 |
+
motion_cache_shard_args+=(--input-already-sharded)
|
| 256 |
+
fi
|
| 257 |
+
"$PYTHON_BIN" cache_video_omni_embeds.py \
|
| 258 |
+
--dataset-root "$MOTION_EXAMPLE_CLIP_ROOT" \
|
| 259 |
+
--metadata-jsonl "$motion_metadata_jsonl" \
|
| 260 |
+
--embed-root "$MOTION_EXAMPLE_CLIP_EMBED_ROOT" \
|
| 261 |
+
--model-path "$MODEL_PATH" \
|
| 262 |
+
--video-sample-fps "$VIDEO_SAMPLE_FPS" \
|
| 263 |
+
--video-size-pixels "$VIDEO_SIZE_PIXELS" \
|
| 264 |
+
--video-decoder "$VIDEO_DECODER" \
|
| 265 |
+
--dtype float16 \
|
| 266 |
+
--num-workers "$EMBED_NUM_WORKERS" \
|
| 267 |
+
--existing-cache-check "$EXISTING_CACHE_CHECK" \
|
| 268 |
+
"${motion_cache_shard_args[@]}" \
|
| 269 |
+
"${limit_args[@]}" \
|
| 270 |
+
"${direct_source_clip_args[@]}" \
|
| 271 |
+
"${embed_compress_args[@]}" \
|
| 272 |
+
"${embed_overwrite_args[@]}"
|
| 273 |
+
fi
|
| 274 |
+
|
| 275 |
+
echo "[DONE] video clip datasets and missing caches are ready."
|
cache_video_omni_embeds_dense_stride1_direct_full.sh
ADDED
|
@@ -0,0 +1,523 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#!/usr/bin/env bash
|
| 2 |
+
set -euo pipefail
|
| 3 |
+
|
| 4 |
+
PROJECT_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/Psi0-yth-v18
|
| 5 |
+
SCRIPT_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/Psi0-yth-v21
|
| 6 |
+
cd "$PROJECT_ROOT"
|
| 7 |
+
|
| 8 |
+
DATASET_ROOT=${PSI_VIDEO_CLIP_DATASET_ROOT:-/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/videogen/psi0_video_clip_50f_dense_stride1_direct_full}
|
| 9 |
+
SHARD_NUM_TOTAL=${SHARD_NUM_TOTAL:-4}
|
| 10 |
+
DATASET_NAMES=${DATASET_NAMES:-all}
|
| 11 |
+
DEFAULT_PSI_PYTHON_ENV=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/Psi0-yth-v15/.venv-psi
|
| 12 |
+
KNOWN_DATASETS=(
|
| 13 |
+
dance_1723_video_10s
|
| 14 |
+
motion_example_0506_demo
|
| 15 |
+
2264_with_audio
|
| 16 |
+
distilled_video_0226
|
| 17 |
+
)
|
| 18 |
+
DATASET_NAME_LIST=()
|
| 19 |
+
|
| 20 |
+
if [ -z "${PSI_PYTHON_BIN:-}" ] && [ -z "${PSI_PYTHON_ENV:-}" ] && [ -x "$DEFAULT_PSI_PYTHON_ENV/bin/python" ]; then
|
| 21 |
+
export PSI_PYTHON_ENV="$DEFAULT_PSI_PYTHON_ENV"
|
| 22 |
+
fi
|
| 23 |
+
|
| 24 |
+
prepare_python_env() {
|
| 25 |
+
PSI_PYTHON_BIN=${PSI_PYTHON_BIN:-}
|
| 26 |
+
PSI_PYTHON_ENV=${PSI_PYTHON_ENV:-}
|
| 27 |
+
if [ -z "$PSI_PYTHON_BIN" ] && [ -n "$PSI_PYTHON_ENV" ]; then
|
| 28 |
+
if [ ! -x "$PSI_PYTHON_ENV/bin/python" ]; then
|
| 29 |
+
echo "ERROR: PSI_PYTHON_ENV does not contain an executable python: $PSI_PYTHON_ENV/bin/python" >&2
|
| 30 |
+
exit 2
|
| 31 |
+
fi
|
| 32 |
+
# shellcheck disable=SC1090
|
| 33 |
+
source "$PSI_PYTHON_ENV/bin/activate"
|
| 34 |
+
fi
|
| 35 |
+
[ -f .env ] && source .env
|
| 36 |
+
export PYTHONPATH=$PROJECT_ROOT/src
|
| 37 |
+
PYTHON_BIN=${PSI_PYTHON_BIN:-$(command -v python)}
|
| 38 |
+
if [ ! -x "$PYTHON_BIN" ]; then
|
| 39 |
+
echo "ERROR: python executable is not valid: $PYTHON_BIN" >&2
|
| 40 |
+
exit 2
|
| 41 |
+
fi
|
| 42 |
+
}
|
| 43 |
+
|
| 44 |
+
metadata_jsonl_for_current_shard() {
|
| 45 |
+
if (( SHARD_NUM_TOTAL > 1 )); then
|
| 46 |
+
printf "metadata.shard-%05d-of-%05d.jsonl" "$SHARD_ID_VALUE" "$SHARD_NUM_TOTAL"
|
| 47 |
+
else
|
| 48 |
+
printf "metadata.jsonl"
|
| 49 |
+
fi
|
| 50 |
+
}
|
| 51 |
+
|
| 52 |
+
build_video_only_dense_metadata() {
|
| 53 |
+
local source_root="$1"
|
| 54 |
+
local output_root="$2"
|
| 55 |
+
local dataset_name="$3"
|
| 56 |
+
local source_metadata_jsonl="${4:-metadata.jsonl}"
|
| 57 |
+
local shard_metadata_jsonl
|
| 58 |
+
shard_metadata_jsonl=$(metadata_jsonl_for_current_shard)
|
| 59 |
+
|
| 60 |
+
mkdir -p "$output_root"
|
| 61 |
+
echo "------------------------------------------"
|
| 62 |
+
echo "Building dense metadata for: ${dataset_name}"
|
| 63 |
+
echo "Source root: ${source_root}"
|
| 64 |
+
echo "Output root: ${output_root}"
|
| 65 |
+
echo "Metadata: ${shard_metadata_jsonl}"
|
| 66 |
+
echo "Shard: ${SHARD_ID_VALUE}/${SHARD_NUM_TOTAL}"
|
| 67 |
+
echo "------------------------------------------"
|
| 68 |
+
|
| 69 |
+
"$PYTHON_BIN" - "$source_root" "$output_root" "$dataset_name" "$source_metadata_jsonl" "$shard_metadata_jsonl" <<'PY'
|
| 70 |
+
import concurrent.futures
|
| 71 |
+
import hashlib
|
| 72 |
+
import json
|
| 73 |
+
import os
|
| 74 |
+
import subprocess
|
| 75 |
+
import sys
|
| 76 |
+
import uuid
|
| 77 |
+
from pathlib import Path
|
| 78 |
+
|
| 79 |
+
source_root = Path(sys.argv[1])
|
| 80 |
+
output_root = Path(sys.argv[2])
|
| 81 |
+
dataset_name = sys.argv[3]
|
| 82 |
+
source_metadata_jsonl = sys.argv[4]
|
| 83 |
+
shard_metadata_jsonl = sys.argv[5]
|
| 84 |
+
|
| 85 |
+
shard_num = int(os.environ.get("SHARD_NUM", os.environ.get("SHARD_NUM_TOTAL", "1")))
|
| 86 |
+
shard_id = int(os.environ.get("SHARD_ID", os.environ.get("SHARD_ID_VALUE", "0")))
|
| 87 |
+
limit = int(os.environ.get("LIMIT", "0"))
|
| 88 |
+
motion_fps = float(os.environ.get("MOTION_FPS", "50"))
|
| 89 |
+
action_chunk_frames = int(os.environ.get("ACTION_CHUNK_FRAMES", "30"))
|
| 90 |
+
chunk_stride_frames = max(1, int(os.environ.get("CHUNK_STRIDE_FRAMES", "1")))
|
| 91 |
+
context_frames = int(os.environ.get("CLIP_FRAMES", "50"))
|
| 92 |
+
clip_selection_mode = os.environ.get("CLIP_SELECTION_MODE", "all")
|
| 93 |
+
clip_random_seed = int(os.environ.get("CLIP_RANDOM_SEED", "42"))
|
| 94 |
+
clip_middle_fraction = float(os.environ.get("CLIP_MIDDLE_FRACTION", "0.8"))
|
| 95 |
+
build_num_workers = max(1, int(os.environ.get("BUILD_NUM_WORKERS", "16")))
|
| 96 |
+
chunk_progress_interval = max(0, int(os.environ.get("CHUNK_PROGRESS_INTERVAL", "0")))
|
| 97 |
+
ffprobe_bin = os.environ.get("PSI_FFPROBE_BIN", "ffprobe")
|
| 98 |
+
default_caption = "Generate a motion sequence that matches the provided video."
|
| 99 |
+
|
| 100 |
+
metadata_path = source_root / source_metadata_jsonl
|
| 101 |
+
if not metadata_path.exists():
|
| 102 |
+
raise FileNotFoundError(f"metadata_jsonl does not exist: {metadata_path}")
|
| 103 |
+
|
| 104 |
+
motion_chunk_frames = action_chunk_frames + 1
|
| 105 |
+
chunk_duration_sec = float(action_chunk_frames) / motion_fps
|
| 106 |
+
output_metadata_path = output_root / shard_metadata_jsonl
|
| 107 |
+
final_metadata_path = output_root / "metadata.jsonl"
|
| 108 |
+
|
| 109 |
+
|
| 110 |
+
def stable_hash_mod(value: str, modulo: int) -> int:
|
| 111 |
+
digest = hashlib.sha1(value.encode("utf-8")).digest()
|
| 112 |
+
return int.from_bytes(digest[:8], "big") % int(modulo)
|
| 113 |
+
|
| 114 |
+
|
| 115 |
+
def shard_key(row: dict) -> str:
|
| 116 |
+
return str(row.get("name") or row.get("id") or row.get("video_path") or row.get("original_video_path") or "")
|
| 117 |
+
|
| 118 |
+
|
| 119 |
+
def parse_duration(row: dict) -> float:
|
| 120 |
+
for key in ("duration", "video_duration_sec"):
|
| 121 |
+
value = row.get(key)
|
| 122 |
+
if isinstance(value, (int, float)) and float(value) > 0:
|
| 123 |
+
return float(value)
|
| 124 |
+
meta_info = row.get("meta_info")
|
| 125 |
+
if isinstance(meta_info, dict):
|
| 126 |
+
value = meta_info.get("meta_duration")
|
| 127 |
+
if isinstance(value, (int, float)) and float(value) > 0:
|
| 128 |
+
return float(value)
|
| 129 |
+
return 0.0
|
| 130 |
+
|
| 131 |
+
|
| 132 |
+
def probe_duration(video_path: Path) -> float:
|
| 133 |
+
cmd = [
|
| 134 |
+
ffprobe_bin,
|
| 135 |
+
"-v",
|
| 136 |
+
"error",
|
| 137 |
+
"-select_streams",
|
| 138 |
+
"v:0",
|
| 139 |
+
"-show_entries",
|
| 140 |
+
"stream=duration:format=duration",
|
| 141 |
+
"-of",
|
| 142 |
+
"json",
|
| 143 |
+
str(video_path),
|
| 144 |
+
]
|
| 145 |
+
proc = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, check=False)
|
| 146 |
+
if proc.returncode != 0:
|
| 147 |
+
raise RuntimeError(proc.stderr.decode("utf-8", errors="ignore").strip())
|
| 148 |
+
info = json.loads(proc.stdout.decode("utf-8"))
|
| 149 |
+
streams = info.get("streams") or []
|
| 150 |
+
duration = 0.0
|
| 151 |
+
if streams:
|
| 152 |
+
duration = float(streams[0].get("duration") or 0.0)
|
| 153 |
+
if duration <= 0:
|
| 154 |
+
duration = float(info.get("format", {}).get("duration") or 0.0)
|
| 155 |
+
if duration <= 0:
|
| 156 |
+
raise RuntimeError(f"bad duration for {video_path}")
|
| 157 |
+
return duration
|
| 158 |
+
|
| 159 |
+
|
| 160 |
+
def caption_for_row(row: dict) -> str:
|
| 161 |
+
for key in ("video_caption", "caption", "instruction", "text"):
|
| 162 |
+
value = row.get(key)
|
| 163 |
+
if isinstance(value, str) and value.strip():
|
| 164 |
+
return value.strip()
|
| 165 |
+
return default_caption
|
| 166 |
+
|
| 167 |
+
|
| 168 |
+
rows = []
|
| 169 |
+
with metadata_path.open("r", encoding="utf-8") as f:
|
| 170 |
+
for line in f:
|
| 171 |
+
line = line.strip()
|
| 172 |
+
if line:
|
| 173 |
+
rows.append(json.loads(line))
|
| 174 |
+
|
| 175 |
+
if limit > 0:
|
| 176 |
+
rows = rows[:limit]
|
| 177 |
+
|
| 178 |
+
selected_rows = [row for row in rows if shard_num <= 1 or stable_hash_mod(shard_key(row), shard_num) == shard_id]
|
| 179 |
+
print(f"[SHARD] build metadata shard={shard_id}/{shard_num} sources={len(selected_rows)}/{len(rows)}")
|
| 180 |
+
|
| 181 |
+
sources = []
|
| 182 |
+
missing_videos = 0
|
| 183 |
+
for row in selected_rows:
|
| 184 |
+
video_value = row.get("video_path") or row.get("original_video") or row.get("real_path")
|
| 185 |
+
if not video_value:
|
| 186 |
+
continue
|
| 187 |
+
source_video_path = source_root / str(video_value)
|
| 188 |
+
if not source_video_path.exists():
|
| 189 |
+
missing_videos += 1
|
| 190 |
+
continue
|
| 191 |
+
base_name = str(row.get("name") or row.get("id") or source_video_path.stem)
|
| 192 |
+
source_id = str(row.get("id") or base_name)
|
| 193 |
+
duration = parse_duration(row)
|
| 194 |
+
sources.append(
|
| 195 |
+
{
|
| 196 |
+
"base_name": base_name,
|
| 197 |
+
"source_id": source_id,
|
| 198 |
+
"source_video_path": source_video_path.resolve(),
|
| 199 |
+
"source_video_duration_sec": duration,
|
| 200 |
+
"video_caption": caption_for_row(row),
|
| 201 |
+
"motion_type": str(row.get("motion_type") or row.get("source") or dataset_name),
|
| 202 |
+
}
|
| 203 |
+
)
|
| 204 |
+
|
| 205 |
+
needs_probe = [idx for idx, source in enumerate(sources) if source["source_video_duration_sec"] <= 0]
|
| 206 |
+
if needs_probe:
|
| 207 |
+
max_workers = min(build_num_workers, 32)
|
| 208 |
+
with concurrent.futures.ThreadPoolExecutor(max_workers=max_workers) as executor:
|
| 209 |
+
future_map = {
|
| 210 |
+
executor.submit(probe_duration, sources[idx]["source_video_path"]): idx
|
| 211 |
+
for idx in needs_probe
|
| 212 |
+
}
|
| 213 |
+
for future in concurrent.futures.as_completed(future_map):
|
| 214 |
+
idx = future_map[future]
|
| 215 |
+
sources[idx]["source_video_duration_sec"] = future.result()
|
| 216 |
+
|
| 217 |
+
tmp_path = output_metadata_path.with_name(f".{output_metadata_path.name}.{os.getpid()}.{uuid.uuid4().hex}.tmp")
|
| 218 |
+
row_count = 0
|
| 219 |
+
skipped_short = 0
|
| 220 |
+
with tmp_path.open("w", encoding="utf-8") as out:
|
| 221 |
+
for source_idx, source in enumerate(sources, start=1):
|
| 222 |
+
duration_sec = float(source["source_video_duration_sec"])
|
| 223 |
+
max_start_frame = int((duration_sec - chunk_duration_sec) * motion_fps)
|
| 224 |
+
if max_start_frame < 0:
|
| 225 |
+
skipped_short += 1
|
| 226 |
+
continue
|
| 227 |
+
chunk_idx = 0
|
| 228 |
+
for chunk_start_frame in range(0, max_start_frame + 1, chunk_stride_frames):
|
| 229 |
+
chunk_name = f"{source['base_name']}.chunk{chunk_idx:02d}.f{chunk_start_frame:06d}"
|
| 230 |
+
row = {
|
| 231 |
+
"id": f"{source['source_id']}.chunk{chunk_idx:02d}.f{chunk_start_frame:06d}",
|
| 232 |
+
"name": chunk_name,
|
| 233 |
+
"converted_path": f"motion/{chunk_name}",
|
| 234 |
+
"video_path": f"videos/{chunk_name}.mp4",
|
| 235 |
+
"video_caption": source["video_caption"],
|
| 236 |
+
"duration": chunk_duration_sec,
|
| 237 |
+
"motion_fps": motion_fps,
|
| 238 |
+
"clip_frames": motion_chunk_frames,
|
| 239 |
+
"context_frames": context_frames,
|
| 240 |
+
"action_chunk_frames": action_chunk_frames,
|
| 241 |
+
"motion_chunk_frames": motion_chunk_frames,
|
| 242 |
+
"chunk_stride_frames": chunk_stride_frames,
|
| 243 |
+
"clip_selection_mode": clip_selection_mode,
|
| 244 |
+
"random_seed": clip_random_seed,
|
| 245 |
+
"middle_fraction": clip_middle_fraction,
|
| 246 |
+
"context_start_frame": chunk_start_frame,
|
| 247 |
+
"chunk_start_frame": chunk_start_frame,
|
| 248 |
+
"chunk_start_sec": float(chunk_start_frame) / motion_fps,
|
| 249 |
+
"source_video_duration_sec": duration_sec,
|
| 250 |
+
"source_video_path": str(source["source_video_path"]),
|
| 251 |
+
"motion_type": source["motion_type"],
|
| 252 |
+
"metadata_only": True,
|
| 253 |
+
}
|
| 254 |
+
out.write(json.dumps(row, ensure_ascii=False) + "\n")
|
| 255 |
+
row_count += 1
|
| 256 |
+
chunk_idx += 1
|
| 257 |
+
if chunk_progress_interval > 0 and source_idx % chunk_progress_interval == 0:
|
| 258 |
+
print(
|
| 259 |
+
f"[BUILD] dataset={dataset_name} shard={shard_id}/{shard_num} "
|
| 260 |
+
f"sources={source_idx}/{len(sources)} rows={row_count}"
|
| 261 |
+
)
|
| 262 |
+
|
| 263 |
+
tmp_path.replace(output_metadata_path)
|
| 264 |
+
print(
|
| 265 |
+
f"[DONE] dataset={dataset_name} metadata_rows={row_count} "
|
| 266 |
+
f"skipped_short={skipped_short} missing_videos={missing_videos} "
|
| 267 |
+
f"output={output_metadata_path}"
|
| 268 |
+
)
|
| 269 |
+
|
| 270 |
+
if shard_num > 1:
|
| 271 |
+
shard_paths = [output_root / f"metadata.shard-{idx:05d}-of-{shard_num:05d}.jsonl" for idx in range(shard_num)]
|
| 272 |
+
if all(path.exists() for path in shard_paths):
|
| 273 |
+
merge_tmp = final_metadata_path.with_name(f".{final_metadata_path.name}.{os.getpid()}.{uuid.uuid4().hex}.tmp")
|
| 274 |
+
with merge_tmp.open("w", encoding="utf-8") as out:
|
| 275 |
+
for path in shard_paths:
|
| 276 |
+
with path.open("r", encoding="utf-8") as src:
|
| 277 |
+
for line in src:
|
| 278 |
+
out.write(line)
|
| 279 |
+
merge_tmp.replace(final_metadata_path)
|
| 280 |
+
print(f"[MERGE] dataset={dataset_name} output={final_metadata_path}")
|
| 281 |
+
PY
|
| 282 |
+
}
|
| 283 |
+
|
| 284 |
+
parse_dataset_names() {
|
| 285 |
+
local raw_names="${1:-all}"
|
| 286 |
+
local token=""
|
| 287 |
+
DATASET_NAME_LIST=()
|
| 288 |
+
raw_names="${raw_names//,/ }"
|
| 289 |
+
for token in $raw_names; do
|
| 290 |
+
if [ "$token" = "all" ]; then
|
| 291 |
+
DATASET_NAME_LIST=(all)
|
| 292 |
+
return
|
| 293 |
+
fi
|
| 294 |
+
[ -n "$token" ] && DATASET_NAME_LIST+=("$token")
|
| 295 |
+
done
|
| 296 |
+
if [ "${#DATASET_NAME_LIST[@]}" -eq 0 ]; then
|
| 297 |
+
DATASET_NAME_LIST=(all)
|
| 298 |
+
fi
|
| 299 |
+
}
|
| 300 |
+
|
| 301 |
+
is_known_dataset() {
|
| 302 |
+
local candidate="$1"
|
| 303 |
+
local known=""
|
| 304 |
+
for known in "${KNOWN_DATASETS[@]}"; do
|
| 305 |
+
if [ "$known" = "$candidate" ]; then
|
| 306 |
+
return 0
|
| 307 |
+
fi
|
| 308 |
+
done
|
| 309 |
+
return 1
|
| 310 |
+
}
|
| 311 |
+
|
| 312 |
+
validate_dataset_names() {
|
| 313 |
+
local invalid=()
|
| 314 |
+
local name=""
|
| 315 |
+
parse_dataset_names "$DATASET_NAMES"
|
| 316 |
+
if [ "${#DATASET_NAME_LIST[@]}" -eq 1 ] && [ "${DATASET_NAME_LIST[0]}" = "all" ]; then
|
| 317 |
+
return
|
| 318 |
+
fi
|
| 319 |
+
for name in "${DATASET_NAME_LIST[@]}"; do
|
| 320 |
+
if ! is_known_dataset "$name"; then
|
| 321 |
+
invalid+=("$name")
|
| 322 |
+
fi
|
| 323 |
+
done
|
| 324 |
+
if [ "${#invalid[@]}" -gt 0 ]; then
|
| 325 |
+
echo "Unknown dataset name(s): ${invalid[*]}" >&2
|
| 326 |
+
echo "Available dataset names: ${KNOWN_DATASETS[*]}" >&2
|
| 327 |
+
exit 2
|
| 328 |
+
fi
|
| 329 |
+
}
|
| 330 |
+
|
| 331 |
+
dataset_selected() {
|
| 332 |
+
local dataset_name="$1"
|
| 333 |
+
local selected=""
|
| 334 |
+
if [ "${#DATASET_NAME_LIST[@]}" -eq 1 ] && [ "${DATASET_NAME_LIST[0]}" = "all" ]; then
|
| 335 |
+
return 0
|
| 336 |
+
fi
|
| 337 |
+
for selected in "${DATASET_NAME_LIST[@]}"; do
|
| 338 |
+
if [ "$selected" = "$dataset_name" ]; then
|
| 339 |
+
return 0
|
| 340 |
+
fi
|
| 341 |
+
done
|
| 342 |
+
return 1
|
| 343 |
+
}
|
| 344 |
+
|
| 345 |
+
run_video_only_cache() {
|
| 346 |
+
local output_root="$1"
|
| 347 |
+
local embed_root="$2"
|
| 348 |
+
local dataset_name="$3"
|
| 349 |
+
local metadata_jsonl
|
| 350 |
+
metadata_jsonl=$(metadata_jsonl_for_current_shard)
|
| 351 |
+
|
| 352 |
+
local shard_args=()
|
| 353 |
+
local limit_args=()
|
| 354 |
+
local embed_overwrite_args=()
|
| 355 |
+
local embed_compress_args=()
|
| 356 |
+
|
| 357 |
+
if (( SHARD_NUM_TOTAL > 1 )); then
|
| 358 |
+
shard_args+=(--shard-num "$SHARD_NUM_TOTAL" --shard-id "$SHARD_ID_VALUE")
|
| 359 |
+
fi
|
| 360 |
+
if [ "${LIMIT:-0}" -gt 0 ]; then
|
| 361 |
+
limit_args+=(--limit "${LIMIT}")
|
| 362 |
+
fi
|
| 363 |
+
if [ "${OVERWRITE_EMBEDS:-0}" = "1" ]; then
|
| 364 |
+
embed_overwrite_args+=(--overwrite)
|
| 365 |
+
fi
|
| 366 |
+
if [ "${EMBED_SAVE_COMPRESSED:-0}" = "1" ]; then
|
| 367 |
+
embed_compress_args+=(--save-compressed)
|
| 368 |
+
fi
|
| 369 |
+
|
| 370 |
+
mkdir -p "$output_root"
|
| 371 |
+
echo "------------------------------------------"
|
| 372 |
+
echo "Caching extra video-only dataset: ${dataset_name}"
|
| 373 |
+
echo "Dataset root: ${output_root}"
|
| 374 |
+
echo "Embed root: ${embed_root}"
|
| 375 |
+
echo "Metadata: ${metadata_jsonl}"
|
| 376 |
+
echo "Shard: ${SHARD_ID_VALUE}/${SHARD_NUM_TOTAL}"
|
| 377 |
+
echo "------------------------------------------"
|
| 378 |
+
|
| 379 |
+
"$PYTHON_BIN" "$SCRIPT_ROOT/cache_video_omni_embeds.py" \
|
| 380 |
+
--dataset-root "$output_root" \
|
| 381 |
+
--metadata-jsonl "$metadata_jsonl" \
|
| 382 |
+
--embed-root "$embed_root" \
|
| 383 |
+
--model-path "$MODEL_PATH" \
|
| 384 |
+
--video-sample-fps "$VIDEO_SAMPLE_FPS" \
|
| 385 |
+
--video-size-pixels "$VIDEO_SIZE_PIXELS" \
|
| 386 |
+
--video-decoder "$VIDEO_DECODER" \
|
| 387 |
+
--dtype float16 \
|
| 388 |
+
--num-workers "$EMBED_NUM_WORKERS" \
|
| 389 |
+
--existing-cache-check "$EXISTING_CACHE_CHECK" \
|
| 390 |
+
"${shard_args[@]}" \
|
| 391 |
+
--input-already-sharded \
|
| 392 |
+
--direct-source-clips \
|
| 393 |
+
"${limit_args[@]}" \
|
| 394 |
+
"${embed_compress_args[@]}" \
|
| 395 |
+
"${embed_overwrite_args[@]}"
|
| 396 |
+
}
|
| 397 |
+
|
| 398 |
+
validate_dataset_names
|
| 399 |
+
|
| 400 |
+
if [ "$#" -ne 0 ]; then
|
| 401 |
+
echo "This script does not take positional arguments. Use SHARD_ID=<id> instead." >&2
|
| 402 |
+
exit 2
|
| 403 |
+
fi
|
| 404 |
+
|
| 405 |
+
SHARD_ID_VALUE="${SHARD_ID:-}"
|
| 406 |
+
if [ -z "$SHARD_ID_VALUE" ]; then
|
| 407 |
+
echo "Please set SHARD_ID in [0, $((SHARD_NUM_TOTAL - 1))]. Example: SHARD_ID=0 DATASET_NAMES=2264_with_audio bash $0" >&2
|
| 408 |
+
exit 2
|
| 409 |
+
fi
|
| 410 |
+
if ! [[ "$SHARD_ID_VALUE" =~ ^[0-9]+$ ]]; then
|
| 411 |
+
echo "Shard id must be an integer in [0, $((SHARD_NUM_TOTAL - 1))]" >&2
|
| 412 |
+
exit 2
|
| 413 |
+
fi
|
| 414 |
+
if [ "$SHARD_ID_VALUE" -ge "$SHARD_NUM_TOTAL" ]; then
|
| 415 |
+
echo "Shard id must be an integer in [0, $((SHARD_NUM_TOTAL - 1))]" >&2
|
| 416 |
+
exit 2
|
| 417 |
+
fi
|
| 418 |
+
|
| 419 |
+
export PSI_VIDEO_CLIP_DATASET_ROOT=$DATASET_ROOT
|
| 420 |
+
export PSI_VIDEO_DEMO_ROOT=${PSI_VIDEO_DEMO_ROOT:-/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/videogen/motion_example_0506_demo}
|
| 421 |
+
EXTRA_VIDEO_2264_ROOT=${PSI_VIDEO_2264_ROOT:-/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/videogen/2264_with_audio}
|
| 422 |
+
EXTRA_VIDEO_2264_OUTPUT_ROOT=${PSI_VIDEO_2264_OUTPUT_ROOT:-$DATASET_ROOT/2264_with_audio}
|
| 423 |
+
EXTRA_VIDEO_2264_EMBED_ROOT=${PSI_VIDEO_2264_EMBED_ROOT:-$EXTRA_VIDEO_2264_OUTPUT_ROOT/embeds}
|
| 424 |
+
EXTRA_VIDEO_DISTILLED_ROOT=${PSI_VIDEO_DISTILLED_ROOT:-/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/videogen/distilled_video_0226}
|
| 425 |
+
EXTRA_VIDEO_DISTILLED_OUTPUT_ROOT=${PSI_VIDEO_DISTILLED_OUTPUT_ROOT:-$DATASET_ROOT/distilled_video_0226}
|
| 426 |
+
EXTRA_VIDEO_DISTILLED_EMBED_ROOT=${PSI_VIDEO_DISTILLED_EMBED_ROOT:-$EXTRA_VIDEO_DISTILLED_OUTPUT_ROOT/embeds}
|
| 427 |
+
|
| 428 |
+
export BUILD_CLIPS=${BUILD_CLIPS:-1}
|
| 429 |
+
export CACHE_EMBEDS=${CACHE_EMBEDS:-1}
|
| 430 |
+
export BUILD_DANCE=${BUILD_DANCE:-1}
|
| 431 |
+
export BUILD_MOTION_EXAMPLE=${BUILD_MOTION_EXAMPLE:-1}
|
| 432 |
+
export CACHE_DANCE=${CACHE_DANCE:-1}
|
| 433 |
+
export CACHE_MOTION_EXAMPLE=${CACHE_MOTION_EXAMPLE:-1}
|
| 434 |
+
export CACHE_2264_WITH_AUDIO=${CACHE_2264_WITH_AUDIO:-1}
|
| 435 |
+
export CACHE_DISTILLED_VIDEO_0226=${CACHE_DISTILLED_VIDEO_0226:-1}
|
| 436 |
+
|
| 437 |
+
if ! dataset_selected "dance_1723_video_10s"; then
|
| 438 |
+
export BUILD_DANCE=0
|
| 439 |
+
export CACHE_DANCE=0
|
| 440 |
+
fi
|
| 441 |
+
|
| 442 |
+
if ! dataset_selected "motion_example_0506_demo"; then
|
| 443 |
+
export BUILD_MOTION_EXAMPLE=0
|
| 444 |
+
export CACHE_MOTION_EXAMPLE=0
|
| 445 |
+
fi
|
| 446 |
+
|
| 447 |
+
if ! dataset_selected "2264_with_audio"; then
|
| 448 |
+
export CACHE_2264_WITH_AUDIO=0
|
| 449 |
+
fi
|
| 450 |
+
|
| 451 |
+
if ! dataset_selected "distilled_video_0226"; then
|
| 452 |
+
export CACHE_DISTILLED_VIDEO_0226=0
|
| 453 |
+
fi
|
| 454 |
+
|
| 455 |
+
export METADATA_ONLY=1
|
| 456 |
+
export DIRECT_SOURCE_CLIPS=1
|
| 457 |
+
export CLIP_SELECTION_MODE=all
|
| 458 |
+
export CHUNK_STRIDE_FRAMES=1
|
| 459 |
+
export ACTION_CHUNK_FRAMES=30
|
| 460 |
+
export MOTION_FPS=50
|
| 461 |
+
export CLIPS_PER_VIDEO=1
|
| 462 |
+
export CHUNK_PROGRESS_INTERVAL=${CHUNK_PROGRESS_INTERVAL:-2000}
|
| 463 |
+
|
| 464 |
+
export BUILD_NUM_WORKERS=${BUILD_NUM_WORKERS:-64}
|
| 465 |
+
export CHUNK_NUM_WORKERS=${CHUNK_NUM_WORKERS:-1}
|
| 466 |
+
# export EMBED_NUM_WORKERS=${EMBED_NUM_WORKERS:-64}
|
| 467 |
+
export EMBED_NUM_WORKERS=${EMBED_NUM_WORKERS:-32}
|
| 468 |
+
export EMBED_SAVE_COMPRESSED=${EMBED_SAVE_COMPRESSED:-0}
|
| 469 |
+
export EXISTING_CACHE_CHECK=${EXISTING_CACHE_CHECK:-size}
|
| 470 |
+
export CLIP_FRAMES=${CLIP_FRAMES:-50}
|
| 471 |
+
export CLIP_RANDOM_SEED=${CLIP_RANDOM_SEED:-42}
|
| 472 |
+
export CLIP_MIDDLE_FRACTION=${CLIP_MIDDLE_FRACTION:-0.8}
|
| 473 |
+
export VIDEO_SAMPLE_FPS=${VIDEO_SAMPLE_FPS:-8}
|
| 474 |
+
export VIDEO_SIZE_PIXELS=${VIDEO_SIZE_PIXELS:-50176}
|
| 475 |
+
export VIDEO_DECODER=${VIDEO_DECODER:-ffmpeg}
|
| 476 |
+
export SHARD_NUM=$SHARD_NUM_TOTAL
|
| 477 |
+
export SHARD_ID=$SHARD_ID_VALUE
|
| 478 |
+
export OVERWRITE_CLIPS=${OVERWRITE_CLIPS:-0}
|
| 479 |
+
export OVERWRITE_EMBEDS=${OVERWRITE_EMBEDS:-0}
|
| 480 |
+
export OMP_NUM_THREADS=${OMP_NUM_THREADS:-1}
|
| 481 |
+
MODEL_PATH=${PSI_PRETRAIN_BACKBONE_PATH:-/cpfs01/VideoGen/public_ckpt/Qwen/Qwen2.5-Omni-3B}
|
| 482 |
+
|
| 483 |
+
echo "=========================================="
|
| 484 |
+
echo "Running direct dense video_clip shard ${SHARD_ID_VALUE}/${SHARD_NUM_TOTAL}"
|
| 485 |
+
echo "Output root: ${DATASET_ROOT}"
|
| 486 |
+
echo "Datasets: ${DATASET_NAMES}"
|
| 487 |
+
echo "=========================================="
|
| 488 |
+
|
| 489 |
+
if [ "$BUILD_DANCE" = "1" ] || [ "$BUILD_MOTION_EXAMPLE" = "1" ] || [ "$CACHE_DANCE" = "1" ] || [ "$CACHE_MOTION_EXAMPLE" = "1" ]; then
|
| 490 |
+
bash cache_video_omni_embeds.sh
|
| 491 |
+
fi
|
| 492 |
+
|
| 493 |
+
if [ "$BUILD_CLIPS" = "1" ] || [ "$CACHE_EMBEDS" = "1" ]; then
|
| 494 |
+
prepare_python_env
|
| 495 |
+
fi
|
| 496 |
+
|
| 497 |
+
if [ "$BUILD_CLIPS" = "1" ] && [ "$CACHE_2264_WITH_AUDIO" = "1" ]; then
|
| 498 |
+
build_video_only_dense_metadata \
|
| 499 |
+
"$EXTRA_VIDEO_2264_ROOT" \
|
| 500 |
+
"$EXTRA_VIDEO_2264_OUTPUT_ROOT" \
|
| 501 |
+
"2264_with_audio"
|
| 502 |
+
fi
|
| 503 |
+
|
| 504 |
+
if [ "$CACHE_EMBEDS" = "1" ] && [ "$CACHE_2264_WITH_AUDIO" = "1" ]; then
|
| 505 |
+
run_video_only_cache \
|
| 506 |
+
"$EXTRA_VIDEO_2264_OUTPUT_ROOT" \
|
| 507 |
+
"$EXTRA_VIDEO_2264_EMBED_ROOT" \
|
| 508 |
+
"2264_with_audio"
|
| 509 |
+
fi
|
| 510 |
+
|
| 511 |
+
if [ "$BUILD_CLIPS" = "1" ] && [ "$CACHE_DISTILLED_VIDEO_0226" = "1" ]; then
|
| 512 |
+
build_video_only_dense_metadata \
|
| 513 |
+
"$EXTRA_VIDEO_DISTILLED_ROOT" \
|
| 514 |
+
"$EXTRA_VIDEO_DISTILLED_OUTPUT_ROOT" \
|
| 515 |
+
"distilled_video_0226"
|
| 516 |
+
fi
|
| 517 |
+
|
| 518 |
+
if [ "$CACHE_EMBEDS" = "1" ] && [ "$CACHE_DISTILLED_VIDEO_0226" = "1" ]; then
|
| 519 |
+
run_video_only_cache \
|
| 520 |
+
"$EXTRA_VIDEO_DISTILLED_OUTPUT_ROOT" \
|
| 521 |
+
"$EXTRA_VIDEO_DISTILLED_EMBED_ROOT" \
|
| 522 |
+
"distilled_video_0226"
|
| 523 |
+
fi
|
cache_video_omni_embeds_dense_stride1_full.sh
ADDED
|
@@ -0,0 +1,63 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#!/usr/bin/env bash
|
| 2 |
+
set -euo pipefail
|
| 3 |
+
|
| 4 |
+
cd /cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/Psi0-yth-v18
|
| 5 |
+
|
| 6 |
+
DATASET_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/videogen/psi0_video_clip_50f_dense_stride1_full
|
| 7 |
+
SHARD_NUM_TOTAL=4
|
| 8 |
+
|
| 9 |
+
if [ "$#" -eq 0 ]; then
|
| 10 |
+
for shard_id in 0 1; do
|
| 11 |
+
echo "=========================================="
|
| 12 |
+
echo "Running dense video_clip shard ${shard_id}/${SHARD_NUM_TOTAL}"
|
| 13 |
+
echo "Output root: ${DATASET_ROOT}"
|
| 14 |
+
echo "=========================================="
|
| 15 |
+
bash "$0" "$shard_id"
|
| 16 |
+
done
|
| 17 |
+
exit 0
|
| 18 |
+
fi
|
| 19 |
+
|
| 20 |
+
SHARD_ID_VALUE="$1"
|
| 21 |
+
if ! [[ "$SHARD_ID_VALUE" =~ ^[0-9]+$ ]]; then
|
| 22 |
+
echo "Shard id must be one of: 0 1" >&2
|
| 23 |
+
exit 2
|
| 24 |
+
fi
|
| 25 |
+
if [ "$SHARD_ID_VALUE" -ge "$SHARD_NUM_TOTAL" ]; then
|
| 26 |
+
echo "Shard id must be one of: 0 1" >&2
|
| 27 |
+
exit 2
|
| 28 |
+
fi
|
| 29 |
+
|
| 30 |
+
export PSI_VIDEO_CLIP_DATASET_ROOT=$DATASET_ROOT
|
| 31 |
+
export PSI_VIDEO_DEMO_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/videogen/motion_example_0506_demo
|
| 32 |
+
|
| 33 |
+
export BUILD_CLIPS=1
|
| 34 |
+
export CACHE_EMBEDS=1
|
| 35 |
+
export BUILD_DANCE=1
|
| 36 |
+
export BUILD_MOTION_EXAMPLE=1
|
| 37 |
+
export CACHE_DANCE=1
|
| 38 |
+
export CACHE_MOTION_EXAMPLE=1
|
| 39 |
+
|
| 40 |
+
export CLIP_SELECTION_MODE=all
|
| 41 |
+
export CHUNK_STRIDE_FRAMES=1
|
| 42 |
+
export ACTION_CHUNK_FRAMES=30
|
| 43 |
+
export MOTION_FPS=50
|
| 44 |
+
export CLIPS_PER_VIDEO=1
|
| 45 |
+
export CHUNK_PROGRESS_INTERVAL=200
|
| 46 |
+
|
| 47 |
+
export BUILD_NUM_WORKERS=64
|
| 48 |
+
export CHUNK_NUM_WORKERS=2
|
| 49 |
+
export FFMPEG_THREADS=2
|
| 50 |
+
export X264_PRESET=veryfast
|
| 51 |
+
export EXISTING_VIDEO_CHECK=size
|
| 52 |
+
export EMBED_NUM_WORKERS=64
|
| 53 |
+
export VIDEO_SAMPLE_FPS=8
|
| 54 |
+
export VIDEO_SIZE_PIXELS=50176
|
| 55 |
+
export VIDEO_DECODER=ffmpeg
|
| 56 |
+
export SHARD_NUM=$SHARD_NUM_TOTAL
|
| 57 |
+
export SHARD_ID=$SHARD_ID_VALUE
|
| 58 |
+
export OVERWRITE_CLIPS=0
|
| 59 |
+
export OVERWRITE_EMBEDS=0
|
| 60 |
+
export OMP_NUM_THREADS=1
|
| 61 |
+
|
| 62 |
+
exec bash cache_video_omni_embeds.sh
|
| 63 |
+
sleep 86400
|
cache_video_omni_embeds_full.sh
ADDED
|
@@ -0,0 +1,31 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#!/usr/bin/env bash
|
| 2 |
+
set -euo pipefail
|
| 3 |
+
|
| 4 |
+
cd /cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/Psi0-yth-v18
|
| 5 |
+
|
| 6 |
+
export PSI_VIDEO_CLIP_DATASET_ROOT=${PSI_VIDEO_CLIP_DATASET_ROOT:-/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/videogen/psi0_video_clip_50f_rand32_full}
|
| 7 |
+
export PSI_VIDEO_DEMO_ROOT=${PSI_VIDEO_DEMO_ROOT:-/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/videogen/motion_example_0506_demo}
|
| 8 |
+
|
| 9 |
+
export BUILD_CLIPS=${BUILD_CLIPS:-1}
|
| 10 |
+
export CACHE_EMBEDS=${CACHE_EMBEDS:-1}
|
| 11 |
+
export BUILD_DANCE=${BUILD_DANCE:-1}
|
| 12 |
+
export BUILD_MOTION_EXAMPLE=${BUILD_MOTION_EXAMPLE:-1}
|
| 13 |
+
export CACHE_DANCE=${CACHE_DANCE:-1}
|
| 14 |
+
export CACHE_MOTION_EXAMPLE=${CACHE_MOTION_EXAMPLE:-1}
|
| 15 |
+
|
| 16 |
+
export CLIP_SELECTION_MODE=all
|
| 17 |
+
export CHUNK_STRIDE_FRAMES=${CHUNK_STRIDE_FRAMES:-20}
|
| 18 |
+
export ACTION_CHUNK_FRAMES=${ACTION_CHUNK_FRAMES:-30}
|
| 19 |
+
export MOTION_FPS=${MOTION_FPS:-50}
|
| 20 |
+
export CLIPS_PER_VIDEO=${CLIPS_PER_VIDEO:-1}
|
| 21 |
+
|
| 22 |
+
export BUILD_NUM_WORKERS=${BUILD_NUM_WORKERS:-256}
|
| 23 |
+
export EMBED_NUM_WORKERS=${EMBED_NUM_WORKERS:-256}
|
| 24 |
+
export VIDEO_SAMPLE_FPS=${VIDEO_SAMPLE_FPS:-32}
|
| 25 |
+
export SHARD_NUM=${SHARD_NUM:-4}
|
| 26 |
+
export SHARD_ID=${SHARD_ID:-0}
|
| 27 |
+
export OVERWRITE_CLIPS=${OVERWRITE_CLIPS:-1}
|
| 28 |
+
export OVERWRITE_EMBEDS=${OVERWRITE_EMBEDS:-1}
|
| 29 |
+
export OMP_NUM_THREADS=${OMP_NUM_THREADS:-1}
|
| 30 |
+
|
| 31 |
+
exec bash cache_video_omni_embeds.sh
|
dataset/seed/g1/csv/220714/jog_backward_start_001__A025.csv
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
dataset/seed/g1/csv/220714/jog_backward_start_003__A025.csv
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
dataset/seed/g1/csv/230215/idle_crouch_right_loop_R_002__A194.csv
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
dataset/seed/g1/csv/230306/crouch_idle_004__A244.csv
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
dataset/seed/g1/csv/240327/jump_twice_R_001__A533.csv
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
dataset/seed/g1/csv/240327/jump_twice_R_002__A533.csv
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
dataset/seed/g1/csv/240327/jump_twice_R_003__A533.csv
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
dataset/seed/g1/csv/240327/touch_toes_standing_R_001__A533.csv
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
dataset/seed/metadata/seed_metadata_v004_100.csv
ADDED
|
@@ -0,0 +1,101 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
move_name,filename,move_duration_frames,package,category,is_neutral,is_mirror,move_soma_uniform_path,move_soma_uniform_shape_path,move_soma_proportional_path,move_soma_proportional_shape_path,move_g1_path,take_name,take_actor,take_org_name,take_date,take_day_part,content_name,content_natural_desc_1,content_natural_desc_2,content_natural_desc_3,content_natural_desc_4,content_technical_description,content_short_description,content_short_description_2,content_all_rigplay_styles,content_uniform_style,content_type_of_movement,content_body_position,content_horizontal_move,content_vertical_move,content_props,content_complex_action,content_repeated_action,actor_uid,actor_height,actor_height_cm,actor_foot_cm,actor_collarbone_height_cm,actor_collarbone_span_cm,actor_elbow_span_cm,actor_wrist_span_cm,actor_shoulder_span_cm,actor_hips_height_cm,actor_hips_bones_span_cm,actor_knee_height_cm,actor_ankle_height_cm,actor_weight_kg,actor_age_yr,actor_gender,actor_profession
|
| 2 |
+
crouch_idle_001__A147,idle_crouch_loop_001__A147,761,Locomotion,Advanced Locomotion,1.0,False,soma_uniform/bvh/230125/idle_crouch_loop_001__A147.bvh,soma_shapes/soma_base_fit_mhr_params.npz,soma_proportional/bvh/230125/idle_crouch_loop_001__A147.bvh,soma_shapes/soma_proportion_fit_mhr_params/A147.npz,g1/csv/230125/idle_crouch_loop_001__A147.csv,crouch_idle_001,A147,idle_crouch_loop_001,230125,_2,crouch_idle,assume a crouching position with feet almost parallel,assumes a squatting posture with nearly parallel feet in an open position,person is crouching down with their feet in a parallel and open position,A person is sitting in a crouched posture.,"feet almost parallel in open position, talent crouching",crouch position,crouch idle,neutral,neutral,walking,crouching,0,0,0,0,0,A147,M,183,28,150,30,92,143,182,100,31,57,10,84,26,M,actor
|
| 3 |
+
high_jump_R_opt_1_001__A280,high_jump_R_001__A280,639,Locomotion,Baseline,1.0,False,soma_uniform/bvh/230316/high_jump_R_001__A280.bvh,soma_shapes/soma_base_fit_mhr_params.npz,soma_proportional/bvh/230316/high_jump_R_001__A280.bvh,soma_shapes/soma_proportion_fit_mhr_params/A280.npz,g1/csv/230316/high_jump_R_001__A280.csv,high_jump_R_opt_1_001,A280,high_jump_R_001,230316,_1,high_jump_R_opt_1,high jump over a moving pole in one place,making high jump over a moving pole stationary,performer makes high jump in one place over a moving pole,A person energetically does a high jump while facing forward.,"jump high over moving pole, jump high in one place",jumping high,high jump opt 1,neutral,neutral,jumping,standing,1,1,0,0,0,A280,M,171,27,141,32,93,134,170,99,27,50,9,68,25,M,actor
|
| 4 |
+
crouch_idle_right_R_001__A194,idle_crouch_right_loop_R_001__A194,1380,Locomotion,Advanced Locomotion,1.0,False,soma_uniform/bvh/230215/idle_crouch_right_loop_R_001__A194.bvh,soma_shapes/soma_base_fit_mhr_params.npz,soma_proportional/bvh/230215/idle_crouch_right_loop_R_001__A194.bvh,soma_shapes/soma_proportion_fit_mhr_params/A194.npz,g1/csv/230215/idle_crouch_right_loop_R_001__A194.csv,crouch_idle_right_R_001,A194,idle_crouch_right_loop_R_001,230215,_1,crouch_idle_right_R,character with right foot forward assumes a crouching position leaning on the left foot,stand in a crouching position with right foot forward,maintain a crouching position with the right leg slightly extended forward,A crouched person is standing in place with their right leg ahead of the left.,"right foot slightly forward, body weight on left foot, talent crouching",crouch position right leg forward,crouch idle right,neutral,neutral,crouching idle,crouching,0,0,0,0,0,A194,M,178,29,146,29,91,138,173,98,32,53,10,90,25,M,actor
|
| 5 |
+
walk_ff_start_315_R_001__A049,walk_ff_start_315_001__A049,692,Locomotion,Basic Locomotion Neutral,1.0,False,soma_uniform/bvh/221011/walk_ff_start_315_001__A049.bvh,soma_shapes/soma_base_fit_mhr_params.npz,soma_proportional/bvh/221011/walk_ff_start_315_001__A049.bvh,soma_shapes/soma_proportion_fit_mhr_params/A049.npz,g1/csv/221011/walk_ff_start_315_001__A049.csv,walk_ff_start_315_R_001,A049,walk_ff_start_315_001,221011,_1,walk_ff_start_315_R,walking backward starting with right foot and moving diagonally with left foot crossing behind,"begin by moving right foot, then move backward diagonally crossing with left foot",person takes a right diagonal step backward,A person starts walking swiftly toward the back right diagonal in a crosswise manner at a moderate pace.,"start right foot, move backward diagonally, left foot back crossing, walk backward diagonally",start walking back right diagonal,walk ff start 315,neutral,neutral,walking,standing,1,0,0,0,0,A049,S,156,23,130,40,84,120,154,88,24,47,9,50,25,F,general
|
| 6 |
+
jog_arc_ccw_stop_001__A029,arc_jog_left_stop_001__A029,739,Locomotion,Basic Locomotion Neutral,1.0,False,soma_uniform/bvh/220720/arc_jog_left_stop_001__A029.bvh,soma_shapes/soma_base_fit_mhr_params.npz,soma_proportional/bvh/220720/arc_jog_left_stop_001__A029.bvh,soma_shapes/soma_proportion_fit_mhr_params/A029.npz,g1/csv/220720/arc_jog_left_stop_001__A029.csv,jog_arc_ccw_stop_001,A029,arc_jog_left_stop_001,220720,_1,jog_arc_ccw_stop,"person jogs in a counterclockwise circle, then abruptly stops",person stops jogging in a counterclockwise arc,stop jogging in a counterclockwise circle,A person jogs actively towards left in a semicircle.,"jogging in a circle, counterclockwise, stop",stop jogging counterclockwise in an arc,jog arc ccw stop,neutral,neutral,jogging,standing,1,1,0,0,0,A029,S,155,24,127,31,85,120,158,80,22,43,8,41,22,F,general
|
| 7 |
+
walk_ff_stop_360_R_001__A017,Stop_backward_Walk_001__A017,7854,Locomotion,Basic Locomotion Neutral,1.0,False,soma_uniform/bvh/220705/Stop_backward_Walk_001__A017.bvh,soma_shapes/soma_base_fit_mhr_params.npz,soma_proportional/bvh/220705/Stop_backward_Walk_001__A017.bvh,soma_shapes/soma_proportion_fit_mhr_params/A017.npz,g1/csv/220705/Stop_backward_Walk_001__A017.csv,walk_ff_stop_360_R_001,A017,Stop_backward_Walk_001,220705,_1,walk_ff_stop_360_R,person stops and walks backwards,stop the action of walking backwards,"individual ends their backward walk, starting with their left foot",The person stops walking backwards.,"end left foot, move backward, walk backward",stop walking backwards,walk ff stop 360,neutral,neutral,walking,standing,1,0,0,0,0,A017,T,180,27,148,43,95,144,178,100,27,50,9,60,32,F,general
|
| 8 |
+
crouch_ff_stop_180_R_001__A195,crouch_ff_stop_180_R_001__A195,1056,Locomotion,Advanced Locomotion,1.0,False,soma_uniform/bvh/230215/crouch_ff_stop_180_R_001__A195.bvh,soma_shapes/soma_base_fit_mhr_params.npz,soma_proportional/bvh/230215/crouch_ff_stop_180_R_001__A195.bvh,soma_shapes/soma_proportion_fit_mhr_params/A195.npz,g1/csv/230215/crouch_ff_stop_180_R_001__A195.csv,crouch_ff_stop_180_R_001,A195,crouch_ff_stop_180_R_001,230215,_2,crouch_ff_stop_180_R,walking in a forwards direction while in a crouching position,a crouching character stops and faces forward after moving forward,start walking forward while maintaining a crouch position,A person walking forward in a crouching position stops walking and stands in the same stance.,"end left foot, move forward, talent crouching, walking in crouch",stopping crouch facing forward,crouch ff stop 180,neutral,neutral,walking,crouching,1,1,0,0,0,A195,T,177,26,147,30,96,138,174,100,27,54,8,64,25,F,actor
|
| 9 |
+
turn_jump_000_001__A022,turn_jump_0000_001__A022,336,Locomotion,Basic Locomotion Neutral,1.0,False,soma_uniform/bvh/220713/turn_jump_0000_001__A022.bvh,soma_shapes/soma_base_fit_mhr_params.npz,soma_proportional/bvh/220713/turn_jump_0000_001__A022.bvh,soma_shapes/soma_proportion_fit_mhr_params/A022.npz,g1/csv/220713/turn_jump_0000_001__A022.csv,turn_jump_000_001,A022,turn_jump_0000_001,220713,_1,turn_jump_000,jumping back over the left shoulder with a full spin,a jump over the left shoulder with a 180-degree turn,person leaps one step while rotating 180 degrees over the left shoulder,A person quickly jumps back over their left shoulder and then stands.,rotate 180 degrees over your left shoulder while jumping one step,turn jump back over left shoulder,turn jump 000,neutral,neutral,"jumping, turning",standing,1,1,0,0,0,A022,T,180,25,152,32,93,135,174,105,28,51,7,62,30,F,general
|
| 10 |
+
jog_ff_start_180_R_001__A021,jog_forward_start_001__A021,675,Locomotion,Baseline,1.0,False,soma_uniform/bvh/220713/jog_forward_start_001__A021.bvh,soma_shapes/soma_base_fit_mhr_params.npz,soma_proportional/bvh/220713/jog_forward_start_001__A021.bvh,soma_shapes/soma_proportion_fit_mhr_params/A021.npz,g1/csv/220713/jog_forward_start_001__A021.csv,jog_ff_start_180_R_001,A021,jog_forward_start_001,220713,_1,jog_ff_start_180_R,start jogging forward,start a forward jog,initiate a forward jog,A person standing on the ground steadily jogs in the forward direction and then stands on the ground in an upright position.,"start right foot, move forward, jog",start jog forward,jog ff start 180,neutral,neutral,jogging,standing,1,1,0,0,0,A021,S,155,23,129,30,85,121,152,85,24,44,7,41,29,F,general
|
| 11 |
+
jog_arc_ccw_start_001__A029,arc_jog_left_start_001__A029,810,Locomotion,Basic Locomotion Neutral,1.0,False,soma_uniform/bvh/220720/arc_jog_left_start_001__A029.bvh,soma_shapes/soma_base_fit_mhr_params.npz,soma_proportional/bvh/220720/arc_jog_left_start_001__A029.bvh,soma_shapes/soma_proportion_fit_mhr_params/A029.npz,g1/csv/220720/arc_jog_left_start_001__A029.csv,jog_arc_ccw_start_001,A029,arc_jog_left_start_001,220720,_1,jog_arc_ccw_start,jogging is initiated in a counterclockwise arc,commence a jog in counterclockwise direction,person starts jogging in a counterclockwise arc,A person standing starts jogging in an anticlockwise arc.,"jogging in a circle, counterclockwise, start",start jogging counterclockwise in an arc,jog arc ccw start,neutral,neutral,jogging,standing,1,1,0,0,0,A029,S,155,24,127,31,85,120,158,80,22,43,8,41,22,F,general
|
| 12 |
+
jog_arc_cw_stop_R_001__A049,jog_arc_cw_stop_001__A049,727,Locomotion,Basic Locomotion Neutral,1.0,False,soma_uniform/bvh/221011/jog_arc_cw_stop_001__A049.bvh,soma_shapes/soma_base_fit_mhr_params.npz,soma_proportional/bvh/221011/jog_arc_cw_stop_001__A049.bvh,soma_shapes/soma_proportion_fit_mhr_params/A049.npz,g1/csv/221011/jog_arc_cw_stop_001__A049.csv,jog_arc_cw_stop_R_001,A049,jog_arc_cw_stop_001,221011,_1,jog_arc_cw_stop_R,"someone stops jogging in a clockwise, arc-like motion",stop and jog clockwise forming a circular path,person finishes jogging in a clockwise arc,"A person is jogging in a clockwise direction, forming an arc after which they stop at a pace.","end left foot, move forward in a circle, clockwise, jog ",stop jogging clockwise in an arc,jog arc cw stop,neutral,neutral,jogging,standing,1,1,0,0,0,A049,S,156,23,130,40,84,120,154,88,24,47,9,50,25,F,general
|
| 13 |
+
walk_ff_stop_180_R_001__A017,Stop_forward_Walk_001__A017,12547,Locomotion,Basic Locomotion Neutral,1.0,False,soma_uniform/bvh/220705/Stop_forward_Walk_001__A017.bvh,soma_shapes/soma_base_fit_mhr_params.npz,soma_proportional/bvh/220705/Stop_forward_Walk_001__A017.bvh,soma_shapes/soma_proportion_fit_mhr_params/A017.npz,g1/csv/220705/Stop_forward_Walk_001__A017.csv,walk_ff_stop_180_R_001,A017,Stop_forward_Walk_001,220705,_1,walk_ff_stop_180_R,stop moving forward by landing on the left foot while walking,"advance forward and stop with a flourish, landing on your left foot","finish walking forward, ending with the left foot",A person standing is walking in the forward direction at a steady pace.,"end left foot, move forward, walk forward, maximum expression","stop walking forward, finish with left leg",walk ff stop 180,neutral,neutral,walking,standing,1,0,0,0,0,A017,T,180,27,148,43,95,144,178,100,27,50,9,60,32,F,general
|
| 14 |
+
turn_walk_000_L_001__A021,turn_start_walk_0000_001__A021,982,Locomotion,Basic Locomotion Neutral,1.0,False,soma_uniform/bvh/220713/turn_start_walk_0000_001__A021.bvh,soma_shapes/soma_base_fit_mhr_params.npz,soma_proportional/bvh/220713/turn_start_walk_0000_001__A021.bvh,soma_shapes/soma_proportion_fit_mhr_params/A021.npz,g1/csv/220713/turn_start_walk_0000_001__A021.csv,turn_walk_000_L_001,A021,turn_start_walk_0000_001,220713,_1,turn_walk_000_L,figure turns 180 degrees over their left shoulder and then walks forward,a quick turn over the left shoulder followed by a forward walk,character turns 180 degrees over left shoulder and moves forward,The person turns back over the left shoulder and walks backward at a normal pace before stopping.,"start left foot, turn left 180 degrees over left shoulder, walk forward, move forward","turns back over left shoulder, walks forward",turn walk 000,neutral,neutral,"walking, turning",standing,1,0,0,0,0,A021,S,155,23,129,30,85,121,152,85,24,44,7,41,29,F,general
|
| 15 |
+
jog_ff_start_000_001__A025,jog_backward_start_001__A025,469,Locomotion,Basic Locomotion Neutral,1.0,False,soma_uniform/bvh/220714/jog_backward_start_001__A025.bvh,soma_shapes/soma_base_fit_mhr_params.npz,soma_proportional/bvh/220714/jog_backward_start_001__A025.bvh,soma_shapes/soma_proportion_fit_mhr_params/A025.npz,g1/csv/220714/jog_backward_start_001__A025.csv,jog_ff_start_000_001,A025,jog_backward_start_001,220714,_1,jog_ff_start_000,"start a backwards jog, leading with the left leg","jog backward, starting with the left leg","person performing a backwards jog, facing forward, initiating movement with the left leg",A standing person starts jogging backward steadily by taking medium-length strides while facing forward.,"face forward, jogging back, left leg first, start",begin jog backwards,jog ff start 000,neutral,neutral,jogging,standing,1,1,0,0,0,A025,T,199,31,165,44,103,157,201,113,28,58,10,102,19,M,general
|
| 16 |
+
jump_ff_000_001__A021,jump_backward_001__A021,665,Locomotion,Basic Locomotion Neutral,1.0,False,soma_uniform/bvh/220713/jump_backward_001__A021.bvh,soma_shapes/soma_base_fit_mhr_params.npz,soma_proportional/bvh/220713/jump_backward_001__A021.bvh,soma_shapes/soma_proportion_fit_mhr_params/A021.npz,g1/csv/220713/jump_backward_001__A021.csv,jump_ff_000_001,A021,jump_backward_001,220713,_1,jump_ff_000,"jumping back, landing on the right foot",jumping backwards and landing on the right leg,"a jump backwards led by the left foot, while constantly looking forwards",A person jumps backward and then stands in an upright posture.,"jumping back, left foot first, head looking forward","jump backwards, finish with right leg",jump ff 000,neutral,neutral,jumping,standing,1,1,0,0,0,A021,S,155,23,129,30,85,121,152,85,24,44,7,41,29,F,general
|
| 17 |
+
jog_ff_start_315_R_001__A050,jog_ff_start_315_001__A050,426,Locomotion,Basic Locomotion Neutral,1.0,False,soma_uniform/bvh/221011/jog_ff_start_315_001__A050.bvh,soma_shapes/soma_base_fit_mhr_params.npz,soma_proportional/bvh/221011/jog_ff_start_315_001__A050.bvh,soma_shapes/soma_proportion_fit_mhr_params/A050.npz,g1/csv/221011/jog_ff_start_315_001__A050.csv,jog_ff_start_315_R_001,A050,jog_ff_start_315_001,221011,_1,jog_ff_start_315_R,person starts jogging backward diagonally with the right foot,person jogs backward diagonally starting with the right foot,person starts a diagonal jog backward,A person swiftly jogs backward along the back-right diagonal.,"start right foot, move backward diagonally, left foot back crossing, jog",start jog back right diagonally,jog ff start 315,neutral,neutral,jogging,standing,1,1,0,0,0,A050,T,182,26,152,40,99,140,180,105,24,53,10,75,31,F,general
|
| 18 |
+
turn_jump_090_001__A023,turn_jump_0090_001__A023,600,Locomotion,Basic Locomotion Neutral,1.0,False,soma_uniform/bvh/220713/turn_jump_0090_001__A023.bvh,soma_shapes/soma_base_fit_mhr_params.npz,soma_proportional/bvh/220713/turn_jump_0090_001__A023.bvh,soma_shapes/soma_proportion_fit_mhr_params/A023.npz,g1/csv/220713/turn_jump_0090_001__A023.csv,turn_jump_090_001,A023,turn_jump_0090_001,220713,_1,turn_jump_090,individual performing a leftward turning jump step,a one-step jump with a 90-degree rotation over the left shoulder,individual turns 90 degrees over their left shoulder while jumping one step,A person swiftly pivots and jumps left with agility.,rotate 90 degrees over your left shoulder while jumping one step,turn jump left,turn jump 090,neutral,neutral,"jumping, turning",standing,1,1,0,0,0,A023,T,190,31,158,42,106,160,195,105,27,54,10,90,46,M,general
|
| 19 |
+
crouch_turn_360_R_001__A126,turn_crouch_360_001__A126,673,Locomotion,Advanced Locomotion,1.0,False,soma_uniform/bvh/230119/turn_crouch_360_001__A126.bvh,soma_shapes/soma_base_fit_mhr_params.npz,soma_proportional/bvh/230119/turn_crouch_360_001__A126.bvh,soma_shapes/soma_proportion_fit_mhr_params/A126.npz,g1/csv/230119/turn_crouch_360_001__A126.csv,crouch_turn_360_R_001,A126,turn_crouch_360_001,230119,_3,crouch_turn_360_R,character crouches and turns back over the right shoulder,character makes a 180 degree turn over the right shoulder while crouching,complete a turn back over the right shoulder while keeping a crouch,A person turns back over the right shoulder in a couching stance.,"start right foot, turn right 180 degrees over right shoulder, 3 steps, talent crouching, walking in crouch",turning back over right shoulder while crouching,crouch turn 360,neutral,neutral,turning,crouching,0,0,0,0,0,A126,M,169,25,136,25,90,130,164,95,31,51,8,63,25,F,actor
|
| 20 |
+
turn_walk_045_L_001__A023,turn_start_walk_0045_001__A023,538,Locomotion,Basic Locomotion Neutral,1.0,False,soma_uniform/bvh/220713/turn_start_walk_0045_001__A023.bvh,soma_shapes/soma_base_fit_mhr_params.npz,soma_proportional/bvh/220713/turn_start_walk_0045_001__A023.bvh,soma_shapes/soma_proportion_fit_mhr_params/A023.npz,g1/csv/220713/turn_start_walk_0045_001__A023.csv,turn_walk_045_L_001,A023,turn_start_walk_0045_001,220713,_1,turn_walk_045_L,character takes strides forward after a 135-degree left turn over the left shoulder,"character makes a 135 degree left turn, walking forward","person takes a left step, turns left 135 degrees and walks forward",A person turns around towards their left back diagonal and starts walking steadily.,"start left foot, turn left 135 degrees over left shoulder, walk forward diagonally",turns left walks forward,turn walk 045,neutral,neutral,"walking, turning",standing,1,0,0,0,0,A023,T,190,31,158,42,106,160,195,105,27,54,10,90,46,M,general
|
| 21 |
+
jump_twice_R_001__A533,jump_twice_R_001__A533,475,Locomotion,Basic Locomotion Neutral,1.0,False,soma_uniform/bvh/240327/jump_twice_R_001__A533.bvh,soma_shapes/soma_base_fit_mhr_params.npz,soma_proportional/bvh/240327/jump_twice_R_001__A533.bvh,soma_shapes/soma_proportion_fit_mhr_params/A533.npz,g1/csv/240327/jump_twice_R_001__A533.csv,jump_twice_R_001,A533,jump_twice_R_001,240327,_1,jump_twice_R,jump and jump again,person jumps forward twice,two big jumps one after the other,"A person steadily bends, jumps twice in the forward direction, then straightens up, and stands in an upright posture.",person jumps forward right leg first and then jumps again,jump twice in a row,jump twice,neutral,neutral,jumping,standing,1,1,0,0,1,A533,M,171,29,141,35,90,134,172,95,26,54,11,71,27,M,actor
|
| 22 |
+
relaxed_fast_walk_ff_180_001__A005,walk_fast_forward_relax_001__A005,2248,Locomotion,Basic Locomotion Styles,0.0,False,soma_uniform/bvh/210707/walk_fast_forward_relax_001__A005.bvh,soma_shapes/soma_base_fit_mhr_params.npz,soma_proportional/bvh/210707/walk_fast_forward_relax_001__A005.bvh,soma_shapes/soma_proportion_fit_mhr_params/A005.npz,g1/csv/210707/walk_fast_forward_relax_001__A005.csv,relaxed_fast_walk_ff_180_001,A005,walk_fast_forward_relax_001,210707,_1,relaxed_fast_walk_ff_180,a casual individual briskly walking forward with a sense of calmness,"person walks at a brisk pace forward, maintaining an open and relaxed posture","walk forward quickly, doing so with a relaxed, open posture","A person with arms raised sideways lowers them and walks forward quickly, turning around during the walk and then stands with their arms raised sideways.","person walks forward, high energy, open position, calm",relaxed fast walk facing forward,relaxed fast walk ff 180,relaxed,neutral,walking,standing,1,0,0,0,0,A005,S,152,25,129,32,85,120,153,92,28,45,7,50,23,F,general
|
| 23 |
+
crouch_ff_stop_360_R_001__A126,crouch_ff_stop_360_001__A126,847,Locomotion,Advanced Locomotion,1.0,False,soma_uniform/bvh/230119/crouch_ff_stop_360_001__A126.bvh,soma_shapes/soma_base_fit_mhr_params.npz,soma_proportional/bvh/230119/crouch_ff_stop_360_001__A126.bvh,soma_shapes/soma_proportion_fit_mhr_params/A126.npz,g1/csv/230119/crouch_ff_stop_360_001__A126.csv,crouch_ff_stop_360_R_001,A126,crouch_ff_stop_360_001,230119,_3,crouch_ff_stop_360_R,person stops crouched walk and turns over their right shoulder,"a crouching person, moves backward and stops over the right shoulder",person crouching walks backwards and stops over their right shoulder,The crouched person walks backward while facing forward with arms swaying.,"end left foot, move backward, talent crouching, walking in crouch",stopping crouch back over right shoulder,crouch ff stop 360,neutral,neutral,walking,crouching,1,1,0,0,0,A126,M,169,25,136,25,90,130,164,95,31,51,8,63,25,F,actor
|
| 24 |
+
high_jump_full_turn_R_opt_1_001__A479,high_jump_full_turn_R_opt_1_001__A479,303,Locomotion,Advanced Locomotion,1.0,False,soma_uniform/bvh/231011/high_jump_full_turn_R_opt_1_001__A479.bvh,soma_shapes/soma_base_fit_mhr_params.npz,soma_proportional/bvh/231011/high_jump_full_turn_R_opt_1_001__A479.bvh,soma_shapes/soma_proportion_fit_mhr_params/A479.npz,g1/csv/231011/high_jump_full_turn_R_opt_1_001__A479.csv,high_jump_full_turn_R_opt_1_001,A479,high_jump_full_turn_R_opt_1_001,231011,_1,high_jump_full_turn_R_opt_1,turning around during high jump up,the performer turns around while jumping high up,the performer turns around during a high jump,A person swiftly turns around while doing a high jump and then lands on the floor standing in a wide stance.,character turns around during a high jump up,turning around during a high jump up and landing,high jump full turn opt 1,neutral,neutral,jumping,standing,0,1,0,0,0,A479,M,183,31,149,32,102,147,187,103,29,55,10,84,27,M,actor
|
| 25 |
+
high_jump_R_opt_2_001__A476,high_jump_R_opt_2_001__A476,374,Locomotion,Basic Locomotion Neutral,1.0,False,soma_uniform/bvh/231011/high_jump_R_opt_2_001__A476.bvh,soma_shapes/soma_base_fit_mhr_params.npz,soma_proportional/bvh/231011/high_jump_R_opt_2_001__A476.bvh,soma_shapes/soma_proportion_fit_mhr_params/A476.npz,g1/csv/231011/high_jump_R_opt_2_001__A476.csv,high_jump_R_opt_2_001,A476,high_jump_R_opt_2_001,231011,_1,high_jump_R_opt_2,character jumps up high in one place,person performs a high jump in place,character jumps high in place,"A person standing upright jumps high in place and then stands in a place, swaying.",jumping high lifting first right then left leg,high jump in place,high jump opt 2,neutral,neutral,jumping,standing,1,1,0,0,0,A476,M,166,28,137,29,95,129,169,99,30,52,11,55,26,F,actor
|
| 26 |
+
jump_around_001__A492,jump_around_001__A492,1364,Locomotion,Advanced Locomotion,1.0,False,soma_uniform/bvh/231024/jump_around_001__A492.bvh,soma_shapes/soma_base_fit_mhr_params.npz,soma_proportional/bvh/231024/jump_around_001__A492.bvh,soma_shapes/soma_proportion_fit_mhr_params/A492.npz,g1/csv/231024/jump_around_001__A492.csv,jump_around_001,A492,jump_around_001,231024,_2,jump_around,jumping with hands in the air,jumping with hands high,jumping around with arms straight,A person is energetically jumping in place with their hands raised in the air.,character jumps with their hands in the air,jumping with hands raised in the air,jump around,neutral,neutral,jumping,"standing, leaning",0,1,0,0,0,A492,M,170,29,141,27,96,136,172,98,27,53,11,62,28,F,actor
|
| 27 |
+
high_jump_ff_180_R_opt_2_001__A476,high_jump_ff_180_R_opt_2_001__A476,497,Locomotion,Basic Locomotion Neutral,1.0,False,soma_uniform/bvh/231011/high_jump_ff_180_R_opt_2_001__A476.bvh,soma_shapes/soma_base_fit_mhr_params.npz,soma_proportional/bvh/231011/high_jump_ff_180_R_opt_2_001__A476.bvh,soma_shapes/soma_proportion_fit_mhr_params/A476.npz,g1/csv/231011/high_jump_ff_180_R_opt_2_001__A476.csv,high_jump_ff_180_R_opt_2_001,A476,high_jump_ff_180_R_opt_2_001,231011,_1,high_jump_ff_180_R_opt_2,character jumps high forward,person executes a dynamic forward jump,character executes forward jump,"A person is standing, then jumps high forward, lands flat, and again stands in a place while swaying their arms.",jumping high forward lifting first right then left leg,jumps high forward,high jump ff 180 opt 2,neutral,neutral,jumping,standing,1,1,0,0,0,A476,M,166,28,137,29,95,129,169,99,30,52,11,55,26,F,actor
|
| 28 |
+
high_jump_ff_180_R_opt_1_001__A476,high_jump_ff_180_R_opt_1_001__A476,413,Locomotion,Advanced Locomotion,1.0,False,soma_uniform/bvh/231011/high_jump_ff_180_R_opt_1_001__A476.bvh,soma_shapes/soma_base_fit_mhr_params.npz,soma_proportional/bvh/231011/high_jump_ff_180_R_opt_1_001__A476.bvh,soma_shapes/soma_proportion_fit_mhr_params/A476.npz,g1/csv/231011/high_jump_ff_180_R_opt_1_001__A476.csv,high_jump_ff_180_R_opt_1_001,A476,high_jump_ff_180_R_opt_1_001,231011,_1,high_jump_ff_180_R_opt_1,jumping high forward two legs at the same time,the performer jumps forward high with both legs,"jumping forward, the performer lifts both legs off the ground",A person standing upright jumps forward.,character jumps high forward with both legs at the same time,performing a high jump forward with both legs,high jump ff 180 opt 1,neutral,neutral,standing,standing,0,1,0,0,0,A476,M,166,28,137,29,95,129,169,99,30,52,11,55,26,F,actor
|
| 29 |
+
walk_ff_start_360_R_fast_001__A447,walk_ff_start_360_R_fast_001__A447,530,Locomotion,Basic Locomotion Neutral,1.0,False,soma_uniform/bvh/230906/walk_ff_start_360_R_fast_001__A447.bvh,soma_shapes/soma_base_fit_mhr_params.npz,soma_proportional/bvh/230906/walk_ff_start_360_R_fast_001__A447.bvh,soma_shapes/soma_proportion_fit_mhr_params/A447.npz,g1/csv/230906/walk_ff_start_360_R_fast_001__A447.csv,walk_ff_start_360_R_fast_001,A447,walk_ff_start_360_R_fast_001,230906,_1,walk_ff_start_360_R_fast,start walking backward at a fast pace,"at the beginning, person starts walking backward at a fast pace, moving in the opposite direction with high speed","it starts with character walking at a rapid speed, moving in the reverse direction","A person is walking backward at a fast pace, taking short steps.",start fast paced walk back over right shoulder while facing forward,start fast walk back over right shoulder,walk ff start 360 fast,neutral,neutral,walking,standing,1,0,0,0,0,A447,M,171,29,141,30,92,133,174,98,28,56,12,63,27,F,actor
|
| 30 |
+
jog_arc_cw_start_R_001__A093,jog_arc_cw_loop_004__A093,526,Locomotion,Basic Locomotion Neutral,1.0,False,soma_uniform/bvh/230102/jog_arc_cw_loop_004__A093.bvh,soma_shapes/soma_base_fit_mhr_params.npz,soma_proportional/bvh/230102/jog_arc_cw_loop_004__A093.bvh,soma_shapes/soma_proportion_fit_mhr_params/A093.npz,g1/csv/230102/jog_arc_cw_loop_004__A093.csv,jog_arc_cw_start_R_001,A093,jog_arc_cw_loop_004,230102,_1,jog_arc_cw_start_R,start jogging in a clockwise motion in an arc,"begin jogging in a clockwise arc, starting with the right foot","with right foot in front, jog in a clockwise circle movement","A person is standing straight, facing forward, and does not move.","start right foot, move forward in a circle, clockwise, jog ",start jogging clockwise in an arc,jog arc cw start,neutral,neutral,jogging,standing,1,1,0,0,0,A093,T,188,32,157,36,103,151,192,109,30,56,10,89,24,M,actor
|
| 31 |
+
jog_ff_start_360_R_normal_pace_001__A451,jog_ff_start_360_R_normal_pace_001__A451,777,Locomotion,Basic Locomotion Neutral,1.0,False,soma_uniform/bvh/230907/jog_ff_start_360_R_normal_pace_001__A451.bvh,soma_shapes/soma_base_fit_mhr_params.npz,soma_proportional/bvh/230907/jog_ff_start_360_R_normal_pace_001__A451.bvh,soma_shapes/soma_proportion_fit_mhr_params/A451.npz,g1/csv/230907/jog_ff_start_360_R_normal_pace_001__A451.csv,jog_ff_start_360_R_normal_pace_001,A451,jog_ff_start_360_R_normal_pace_001,230907,_1,jog_ff_start_360_R_normal_pace,start jogging backward at a normal pace,"at the beginning, person starts jogging backward at a normal pace, running in reverse with a moderate speed","it starts with character jogging at a regular pace, moving backward with a moderate speed",A person standing upright jogs backwards with small steps at a steady pace.,start normal paced jog back over right shoulder while facing forward,start normal pace jog back over right shoulder,jog ff start 360 normal pace,neutral,neutral,jogging,standing,1,0,0,0,0,A451,M,167,27,136,29,90,131,165,94,26,51,10,50,24,F,actor
|
| 32 |
+
jog_arc_cw_start_R_normal_pace_001__A451,jog_arc_cw_start_R_normal_pace_001__A451,565,Locomotion,Basic Locomotion Neutral,1.0,False,soma_uniform/bvh/230907/jog_arc_cw_start_R_normal_pace_001__A451.bvh,soma_shapes/soma_base_fit_mhr_params.npz,soma_proportional/bvh/230907/jog_arc_cw_start_R_normal_pace_001__A451.bvh,soma_shapes/soma_proportion_fit_mhr_params/A451.npz,g1/csv/230907/jog_arc_cw_start_R_normal_pace_001__A451.csv,jog_arc_cw_start_R_normal_pace_001,A451,jog_arc_cw_start_R_normal_pace_001,230907,_1,jog_arc_cw_start_R_normal_pace,start jogging in an arc at a normal pace,"at the beginning, person starts jogging in an arc at a normal pace, running in a curved path","it begins with character jogging at a regular pace, moving in a curved path",A person jogs in a clockwise circle at a steady pace.,start normal paced jog in clockwise circle starting with right leg,start normal pace jog in clockwise circle,jog arc cw start normal pace,neutral,neutral,jogging,standing,1,0,0,0,0,A451,M,167,27,136,29,90,131,165,94,26,51,10,50,24,F,actor
|
| 33 |
+
walk_ff_start_360_R_normal_pace_001__A445,walk_ff_start_360_R_normal_pace_001__A445,846,Locomotion,Basic Locomotion Neutral,1.0,False,soma_uniform/bvh/230905/walk_ff_start_360_R_normal_pace_001__A445.bvh,soma_shapes/soma_base_fit_mhr_params.npz,soma_proportional/bvh/230905/walk_ff_start_360_R_normal_pace_001__A445.bvh,soma_shapes/soma_proportion_fit_mhr_params/A445.npz,g1/csv/230905/walk_ff_start_360_R_normal_pace_001__A445.csv,walk_ff_start_360_R_normal_pace_001,A445,walk_ff_start_360_R_normal_pace_001,230905,_2,walk_ff_start_360_R_normal_pace,start walking backward at a normal pace,"at the beginning, person starts walking backward at a normal pace, moving in the opposite direction","it starts with character walking at a regular speed, moving in the reverse direction",A person is walking backward at a consistent pace.,start normal paced walk back over right shoulder while facing forward,start normal pace walk back over right shoulder,walk ff start 360 normal pace,neutral,neutral,walking,standing,1,0,0,0,0,A445,M,184,30,150,33,100,149,187,99,28,57,10,74,27,M,actor
|
| 34 |
+
walk_arc_cw_start_R_normal_pace_001__A443,walk_arc_cw_start_R_normal_pace_001__A443,1122,Locomotion,Basic Locomotion Neutral,1.0,False,soma_uniform/bvh/230905/walk_arc_cw_start_R_normal_pace_001__A443.bvh,soma_shapes/soma_base_fit_mhr_params.npz,soma_proportional/bvh/230905/walk_arc_cw_start_R_normal_pace_001__A443.bvh,soma_shapes/soma_proportion_fit_mhr_params/A443.npz,g1/csv/230905/walk_arc_cw_start_R_normal_pace_001__A443.csv,walk_arc_cw_start_R_normal_pace_001,A443,walk_arc_cw_start_R_normal_pace_001,230905,_2,walk_arc_cw_start_R_normal_pace,start walking in an arc at a normal pace,"at the beginning, person starts walking in an arc at a normal pace, moving in a curved path","it begins with character walking at a regular speed, moving in a curved path",A person is walking in a clockwise circular direction at a normal pace.,start normal paced walk in clockwise circle starting with right leg,start normal pace walk in clockwise circle,walk arc cw start normal pace,neutral,neutral,walking,standing,1,0,0,0,0,A443,M,167,27,136,29,90,131,165,94,26,51,10,50,24,F,actor
|
| 35 |
+
walk_ff_start_360_R_very_fast_001__A447,walk_ff_start_360_R_very_fast_001__A447,475,Locomotion,Basic Locomotion Neutral,1.0,False,soma_uniform/bvh/230906/walk_ff_start_360_R_very_fast_001__A447.bvh,soma_shapes/soma_base_fit_mhr_params.npz,soma_proportional/bvh/230906/walk_ff_start_360_R_very_fast_001__A447.bvh,soma_shapes/soma_proportion_fit_mhr_params/A447.npz,g1/csv/230906/walk_ff_start_360_R_very_fast_001__A447.csv,walk_ff_start_360_R_very_fast_001,A447,walk_ff_start_360_R_very_fast_001,230906,_1,walk_ff_start_360_R_very_fast,start walking backward at a very fast pace,individual starts walking backward at a very fast pace,person begins a rapid walk in the backward direction at an extremely high speed,A person standing on the ground is quickly walking in the backward direction while facing forward.,start very fast paced walk back over right shoulder while facing forward,start very fast walk back over right shoulder,walk ff start 360 very fast,neutral,neutral,walking,standing,1,0,0,0,0,A447,M,171,29,141,30,92,133,174,98,28,56,12,63,27,F,actor
|
| 36 |
+
walk_arc_cw_start_R_very_fast_002__A447,walk_arc_cw_start_R_very_fast_002__A447,712,Locomotion,Basic Locomotion Neutral,1.0,False,soma_uniform/bvh/230906/walk_arc_cw_start_R_very_fast_002__A447.bvh,soma_shapes/soma_base_fit_mhr_params.npz,soma_proportional/bvh/230906/walk_arc_cw_start_R_very_fast_002__A447.bvh,soma_shapes/soma_proportion_fit_mhr_params/A447.npz,g1/csv/230906/walk_arc_cw_start_R_very_fast_002__A447.csv,walk_arc_cw_start_R_very_fast_002,A447,walk_arc_cw_start_R_very_fast_002,230906,_1,walk_arc_cw_start_R_very_fast,start walking in an arc at a very fast pace,individual starts walking in an arc at a very fast pace,person begins a fast-paced walk in a curved pattern at an extremely high speed,A person walks in a clockwise circle at a fast pace.,start very fast walk in clockwise circle starting with right leg,start very fast walk in clockwise circle,walk arc cw start very fast,neutral,neutral,walking,standing,1,0,0,0,0,A447,M,171,29,141,30,92,133,174,98,28,56,12,63,27,F,actor
|
| 37 |
+
jog_arc_cw_stop_R_fast_001__A452,jog_arc_cw_stop_R_fast_001__A452,569,Locomotion,Basic Locomotion Neutral,1.0,False,soma_uniform/bvh/230907/jog_arc_cw_stop_R_fast_001__A452.bvh,soma_shapes/soma_base_fit_mhr_params.npz,soma_proportional/bvh/230907/jog_arc_cw_stop_R_fast_001__A452.bvh,soma_shapes/soma_proportion_fit_mhr_params/A452.npz,g1/csv/230907/jog_arc_cw_stop_R_fast_001__A452.csv,jog_arc_cw_stop_R_fast_001,A452,jog_arc_cw_stop_R_fast_001,230907,_1,jog_arc_cw_stop_R_fast,stop jogging in an arc at a fast pace,person ends their fast-paced jog in an arc,individual stops jogging in a curved path at a high speed,"A person jogs quickly in a clockwise circle, then stops.",stop fast jog in clockwise circle ending with right leg,stop fast jog in clockwise circle,jog arc cw stop fast,neutral,neutral,jogging,standing,1,0,0,0,0,A452,M,171,29,141,30,92,133,174,98,28,56,12,63,27,F,actor
|
| 38 |
+
walk_arc_cw_stop_R_fast_002__A447,walk_arc_cw_stop_R_fast_002__A447,739,Locomotion,Basic Locomotion Neutral,1.0,False,soma_uniform/bvh/230906/walk_arc_cw_stop_R_fast_002__A447.bvh,soma_shapes/soma_base_fit_mhr_params.npz,soma_proportional/bvh/230906/walk_arc_cw_stop_R_fast_002__A447.bvh,soma_shapes/soma_proportion_fit_mhr_params/A447.npz,g1/csv/230906/walk_arc_cw_stop_R_fast_002__A447.csv,walk_arc_cw_stop_R_fast_002,A447,walk_arc_cw_stop_R_fast_002,230906,_1,walk_arc_cw_stop_R_fast,stop walking in an arc at a fast pace,person ends their fast-paced walk in a curved pattern,the individual stops walking in an arc at a high speed,A person walks at a fast pace in a clockwise circle and then stops and stands still.,stop fast walk in clockwise circle ending with right leg,stop fast walk in clockwise circle,walk arc cw stop fast,neutral,neutral,walking,standing,1,0,0,0,0,A447,M,171,29,141,30,92,133,174,98,28,56,12,63,27,F,actor
|
| 39 |
+
jog_arc_cw_stop_R_very_fast_001__A455,jog_arc_cw_stop_R_very_fast_001__A455,486,Locomotion,Basic Locomotion Neutral,1.0,False,soma_uniform/bvh/230907/jog_arc_cw_stop_R_very_fast_001__A455.bvh,soma_shapes/soma_base_fit_mhr_params.npz,soma_proportional/bvh/230907/jog_arc_cw_stop_R_very_fast_001__A455.bvh,soma_shapes/soma_proportion_fit_mhr_params/A455.npz,g1/csv/230907/jog_arc_cw_stop_R_very_fast_001__A455.csv,jog_arc_cw_stop_R_very_fast_001,A455,jog_arc_cw_stop_R_very_fast_001,230907,_2,jog_arc_cw_stop_R_very_fast,stop running in an arc at a very fast pace,person is walking forward slowly with a happy expression,character walks happily in a slow pace towards the front,A person quickly runs in a circular path in a clockwise direction towards their right and halts abruptly.,stop very fast jog in clockwise circle ending with right leg,stop very fast jog in clockwise circle,jog arc cw stop very fast,neutral,neutral,jogging,standing,1,0,0,0,0,A455,M,172,26,140,33,91,131,172,95,27,51,10,68,28,F,actor
|
| 40 |
+
walk_ff_stop_360_R_very_fast_002__A447,walk_ff_stop_360_R_very_fast_002__A447,668,Locomotion,Basic Locomotion Neutral,1.0,False,soma_uniform/bvh/230906/walk_ff_stop_360_R_very_fast_002__A447.bvh,soma_shapes/soma_base_fit_mhr_params.npz,soma_proportional/bvh/230906/walk_ff_stop_360_R_very_fast_002__A447.bvh,soma_shapes/soma_proportion_fit_mhr_params/A447.npz,g1/csv/230906/walk_ff_stop_360_R_very_fast_002__A447.csv,walk_ff_stop_360_R_very_fast_002,A447,walk_ff_stop_360_R_very_fast_002,230906,_1,walk_ff_stop_360_R_very_fast,stop walking backward at a very fast pace,person stops their fast-paced walk backward at a very high speed,character ends their quick walk backward at an extremely rapid pace,"A person walks back fast, facing forward, stops, and stands idle.",stop very fast paced walk back over right shoulder while facing forward,stop very fast walk back over right shoulder,walk ff stop 360 very fast,neutral,neutral,walking,standing,1,0,0,0,0,A447,M,171,29,141,30,92,133,174,98,28,56,12,63,27,F,actor
|
| 41 |
+
touch_toes_standing_R_001__A533,touch_toes_standing_R_001__A533,437,Locomotion,Basic Locomotion Neutral,1.0,False,soma_uniform/bvh/240327/touch_toes_standing_R_001__A533.bvh,soma_shapes/soma_base_fit_mhr_params.npz,soma_proportional/bvh/240327/touch_toes_standing_R_001__A533.bvh,soma_shapes/soma_proportion_fit_mhr_params/A533.npz,g1/csv/240327/touch_toes_standing_R_001__A533.csv,touch_toes_standing_R_001,A533,touch_toes_standing_R_001,240327,_1,touch_toes_standing_R,touch your toes while standing,touch your toes,bend over and touch your toes,A person gradually bends forward and touches their toes with their hands while standing.,person standing bends over forward and touches their toes,touch your toes while standing,touch toes standing,neutral,neutral,action,standing,1,0,0,0,0,A533,M,171,29,141,35,90,134,172,95,26,54,11,71,27,M,actor
|
| 42 |
+
jump_ff_360_R_002__A045,jump_ff_360_002__A045,780,Locomotion,Baseline,1.0,False,soma_uniform/bvh/221010/jump_ff_360_002__A045.bvh,soma_shapes/soma_base_fit_mhr_params.npz,soma_proportional/bvh/221010/jump_ff_360_002__A045.bvh,soma_shapes/soma_proportion_fit_mhr_params/A045.npz,g1/csv/221010/jump_ff_360_002__A045.csv,jump_ff_360_R_002,A045,jump_ff_360_002,221010,_1,jump_ff_360_R,character performs a small jump while moving backward,"begin by moving the right foot, then jump backward",person takes a small jump backward,A person jumps backward while facing forward.,"start right foot, move backward, small jump",jump backwards,jump ff 360,neutral,neutral,jumping,standing,1,1,0,0,0,A045,S,156,25,130,39,82,123,154,87,28,45,8,59,22,F,general
|
| 43 |
+
crouch_idle_002__A148,idle_crouch_loop_002__A148,891,Locomotion,Advanced Locomotion,1.0,False,soma_uniform/bvh/230125/idle_crouch_loop_002__A148.bvh,soma_shapes/soma_base_fit_mhr_params.npz,soma_proportional/bvh/230125/idle_crouch_loop_002__A148.bvh,soma_shapes/soma_proportion_fit_mhr_params/A148.npz,g1/csv/230125/idle_crouch_loop_002__A148.csv,crouch_idle_002,A148,idle_crouch_loop_002,230125,_2,crouch_idle,assume a crouching position with feet almost parallel,assumes a squatting posture with nearly parallel feet in an open position,person is crouching down with their feet in a parallel and open position,A person is sitting in a crouched posture.,"feet almost parallel in open position, talent crouching",crouch position,crouch idle,neutral,neutral,walking,crouching,0,0,0,0,0,A148,M,181,29,152,35,99,145,188,103,29,58,10,73,25,M,actor
|
| 44 |
+
high_jump_R_opt_1_002__A280,high_jump_R_002__A280,376,Locomotion,Baseline,1.0,False,soma_uniform/bvh/230316/high_jump_R_002__A280.bvh,soma_shapes/soma_base_fit_mhr_params.npz,soma_proportional/bvh/230316/high_jump_R_002__A280.bvh,soma_shapes/soma_proportion_fit_mhr_params/A280.npz,g1/csv/230316/high_jump_R_002__A280.csv,high_jump_R_opt_1_002,A280,high_jump_R_002,230316,_1,high_jump_R_opt_1,high jump over a moving pole in one place,making high jump over a moving pole stationary,performer makes high jump in one place over a moving pole,A person energetically does a high jump while facing forward.,"jump high over moving pole, jump high in one place",jumping high,high jump opt 1,neutral,neutral,jumping,standing,1,1,0,0,0,A280,M,171,27,141,32,93,134,170,99,27,50,9,68,25,M,actor
|
| 45 |
+
crouch_idle_right_R_002__A194,idle_crouch_right_loop_R_002__A194,1905,Locomotion,Advanced Locomotion,1.0,False,soma_uniform/bvh/230215/idle_crouch_right_loop_R_002__A194.bvh,soma_shapes/soma_base_fit_mhr_params.npz,soma_proportional/bvh/230215/idle_crouch_right_loop_R_002__A194.bvh,soma_shapes/soma_proportion_fit_mhr_params/A194.npz,g1/csv/230215/idle_crouch_right_loop_R_002__A194.csv,crouch_idle_right_R_002,A194,idle_crouch_right_loop_R_002,230215,_1,crouch_idle_right_R,character with right foot forward assumes a crouching position leaning on the left foot,stand in a crouching position with right foot forward,maintain a crouching position with the right leg slightly extended forward,"A person maintains a crouched position with their right leg forward, making slight movements with their forearms.","right foot slightly forward, body weight on left foot, talent crouching",crouch position right leg forward,crouch idle right,neutral,neutral,crouching idle,crouching,0,0,0,0,0,A194,M,178,29,146,29,91,138,173,98,32,53,10,90,25,M,actor
|
| 46 |
+
walk_ff_start_315_R_002__A049,walk_ff_start_315_002__A049,760,Locomotion,Basic Locomotion Neutral,1.0,False,soma_uniform/bvh/221011/walk_ff_start_315_002__A049.bvh,soma_shapes/soma_base_fit_mhr_params.npz,soma_proportional/bvh/221011/walk_ff_start_315_002__A049.bvh,soma_shapes/soma_proportion_fit_mhr_params/A049.npz,g1/csv/221011/walk_ff_start_315_002__A049.csv,walk_ff_start_315_R_002,A049,walk_ff_start_315_002,221011,_1,walk_ff_start_315_R,walking backward starting with right foot and moving diagonally with left foot crossing behind,"begin by moving right foot, then move backward diagonally crossing with left foot",person takes a right diagonal step backward,A person starts walking swiftly toward the back right diagonal in a crosswise manner at a moderate pace.,"start right foot, move backward diagonally, left foot back crossing, walk backward diagonally",start walking back right diagonal,walk ff start 315,neutral,neutral,walking,standing,1,0,0,0,0,A049,S,156,23,130,40,84,120,154,88,24,47,9,50,25,F,general
|
| 47 |
+
jog_arc_ccw_stop_002__A029,arc_jog_left_stop_002__A029,716,Locomotion,Basic Locomotion Neutral,1.0,False,soma_uniform/bvh/220720/arc_jog_left_stop_002__A029.bvh,soma_shapes/soma_base_fit_mhr_params.npz,soma_proportional/bvh/220720/arc_jog_left_stop_002__A029.bvh,soma_shapes/soma_proportion_fit_mhr_params/A029.npz,g1/csv/220720/arc_jog_left_stop_002__A029.csv,jog_arc_ccw_stop_002,A029,arc_jog_left_stop_002,220720,_1,jog_arc_ccw_stop,"person jogs in a counterclockwise circle, then abruptly stops",person stops jogging in a counterclockwise arc,stop jogging in a counterclockwise circle,A person jogs actively towards left in a semicircle.,"jogging in a circle, counterclockwise, stop",stop jogging counterclockwise in an arc,jog arc ccw stop,neutral,neutral,jogging,standing,1,1,0,0,0,A029,S,155,24,127,31,85,120,158,80,22,43,8,41,22,F,general
|
| 48 |
+
walk_ff_stop_360_R_002__A049,walk_ff_stop_360_002__A049,829,Locomotion,Basic Locomotion Neutral,1.0,False,soma_uniform/bvh/221011/walk_ff_stop_360_002__A049.bvh,soma_shapes/soma_base_fit_mhr_params.npz,soma_proportional/bvh/221011/walk_ff_stop_360_002__A049.bvh,soma_shapes/soma_proportion_fit_mhr_params/A049.npz,g1/csv/221011/walk_ff_stop_360_002__A049.csv,walk_ff_stop_360_R_002,A049,walk_ff_stop_360_002,221011,_1,walk_ff_stop_360_R,person stops and walks backwards,stop the action of walking backwards,"individual ends their backward walk, starting with their left foot",The person stops walking backwards.,"end left foot, move backward, walk backward",stop walking backwards,walk ff stop 360,neutral,neutral,walking,standing,1,0,0,0,0,A049,S,156,23,130,40,84,120,154,88,24,47,9,50,25,F,general
|
| 49 |
+
crouch_ff_stop_180_R_002__A147,crouch_ff_stop_180_R_002__A147,734,Locomotion,Advanced Locomotion,1.0,False,soma_uniform/bvh/230125/crouch_ff_stop_180_R_002__A147.bvh,soma_shapes/soma_base_fit_mhr_params.npz,soma_proportional/bvh/230125/crouch_ff_stop_180_R_002__A147.bvh,soma_shapes/soma_proportion_fit_mhr_params/A147.npz,g1/csv/230125/crouch_ff_stop_180_R_002__A147.csv,crouch_ff_stop_180_R_002,A147,crouch_ff_stop_180_R_002,230125,_2,crouch_ff_stop_180_R,walking in a forwards direction while in a crouching position,a crouching character stops and faces forward after moving forward,start walking forward while maintaining a crouch position,A person walking forward in a crouching position stops walking and stands in the same stance.,"end left foot, move forward, talent crouching, walking in crouch",stopping crouch facing forward,crouch ff stop 180,neutral,neutral,walking,crouching,1,1,0,0,0,A147,M,183,28,150,30,92,143,182,100,31,57,10,84,26,M,actor
|
| 50 |
+
turn_jump_000_002__A022,turn_jump_0000_002__A022,326,Locomotion,Basic Locomotion Neutral,1.0,False,soma_uniform/bvh/220713/turn_jump_0000_002__A022.bvh,soma_shapes/soma_base_fit_mhr_params.npz,soma_proportional/bvh/220713/turn_jump_0000_002__A022.bvh,soma_shapes/soma_proportion_fit_mhr_params/A022.npz,g1/csv/220713/turn_jump_0000_002__A022.csv,turn_jump_000_002,A022,turn_jump_0000_002,220713,_1,turn_jump_000,jumping back over the left shoulder with a full spin,a jump over the left shoulder with a 180-degree turn,person leaps one step while rotating 180 degrees over the left shoulder,A person quickly jumps back over their left shoulder and then stands.,rotate 180 degrees over your left shoulder while jumping one step,turn jump back over left shoulder,turn jump 000,neutral,neutral,"jumping, turning",standing,1,1,0,0,0,A022,T,180,25,152,32,93,135,174,105,28,51,7,62,30,F,general
|
| 51 |
+
jog_ff_start_180_R_002__A021,jog_forward_start_002__A021,608,Locomotion,Baseline,1.0,False,soma_uniform/bvh/220713/jog_forward_start_002__A021.bvh,soma_shapes/soma_base_fit_mhr_params.npz,soma_proportional/bvh/220713/jog_forward_start_002__A021.bvh,soma_shapes/soma_proportion_fit_mhr_params/A021.npz,g1/csv/220713/jog_forward_start_002__A021.csv,jog_ff_start_180_R_002,A021,jog_forward_start_002,220713,_1,jog_ff_start_180_R,start jogging forward,start a forward jog,initiate a forward jog,A person standing on the ground steadily jogs in the forward direction and then stands on the ground in an upright position.,"start right foot, move forward, jog",start jog forward,jog ff start 180,neutral,neutral,jogging,standing,1,1,0,0,0,A021,S,155,23,129,30,85,121,152,85,24,44,7,41,29,F,general
|
| 52 |
+
jog_arc_ccw_start_002__A029,arc_jog_left_start_002__A029,403,Locomotion,Basic Locomotion Neutral,1.0,False,soma_uniform/bvh/220720/arc_jog_left_start_002__A029.bvh,soma_shapes/soma_base_fit_mhr_params.npz,soma_proportional/bvh/220720/arc_jog_left_start_002__A029.bvh,soma_shapes/soma_proportion_fit_mhr_params/A029.npz,g1/csv/220720/arc_jog_left_start_002__A029.csv,jog_arc_ccw_start_002,A029,arc_jog_left_start_002,220720,_1,jog_arc_ccw_start,jogging is initiated in a counterclockwise arc,commence a jog in counterclockwise direction,person starts jogging in a counterclockwise arc,A person standing starts jogging in an anticlockwise arc.,"jogging in a circle, counterclockwise, start",start jogging counterclockwise in an arc,jog arc ccw start,neutral,neutral,jogging,standing,1,1,0,0,0,A029,S,155,24,127,31,85,120,158,80,22,43,8,41,22,F,general
|
| 53 |
+
jog_arc_cw_stop_R_002__A045,jog_arc_cw_stop_002__A045,517,Locomotion,Basic Locomotion Neutral,1.0,False,soma_uniform/bvh/221010/jog_arc_cw_stop_002__A045.bvh,soma_shapes/soma_base_fit_mhr_params.npz,soma_proportional/bvh/221010/jog_arc_cw_stop_002__A045.bvh,soma_shapes/soma_proportion_fit_mhr_params/A045.npz,g1/csv/221010/jog_arc_cw_stop_002__A045.csv,jog_arc_cw_stop_R_002,A045,jog_arc_cw_stop_002,221010,_2,jog_arc_cw_stop_R,"someone stops jogging in a clockwise, arc-like motion",stop and jog clockwise forming a circular path,person finishes jogging in a clockwise arc,"A person is jogging in a clockwise direction, forming an arc after which they stop at a pace.","end left foot, move forward in a circle, clockwise, jog ",stop jogging clockwise in an arc,jog arc cw stop,neutral,neutral,jogging,standing,1,1,0,0,0,A045,S,156,25,130,39,82,123,154,87,28,45,8,59,22,F,general
|
| 54 |
+
walk_ff_stop_180_R_002__A023,walk_forward_stop_002__A023,643,Locomotion,Basic Locomotion Neutral,1.0,False,soma_uniform/bvh/220713/walk_forward_stop_002__A023.bvh,soma_shapes/soma_base_fit_mhr_params.npz,soma_proportional/bvh/220713/walk_forward_stop_002__A023.bvh,soma_shapes/soma_proportion_fit_mhr_params/A023.npz,g1/csv/220713/walk_forward_stop_002__A023.csv,walk_ff_stop_180_R_002,A023,walk_forward_stop_002,220713,_1,walk_ff_stop_180_R,stop moving forward by landing on the left foot while walking,"advance forward and stop with a flourish, landing on your left foot","finish walking forward, ending with the left foot",A person standing is walking in the forward direction at a steady pace.,"end left foot, move forward, walk forward, maximum expression","stop walking forward, finish with left leg",walk ff stop 180,neutral,neutral,walking,standing,1,0,0,0,0,A023,T,190,31,158,42,106,160,195,105,27,54,10,90,46,M,general
|
| 55 |
+
turn_walk_000_L_002__A021,turn_start_walk_0000_002__A021,481,Locomotion,Basic Locomotion Neutral,1.0,False,soma_uniform/bvh/220713/turn_start_walk_0000_002__A021.bvh,soma_shapes/soma_base_fit_mhr_params.npz,soma_proportional/bvh/220713/turn_start_walk_0000_002__A021.bvh,soma_shapes/soma_proportion_fit_mhr_params/A021.npz,g1/csv/220713/turn_start_walk_0000_002__A021.csv,turn_walk_000_L_002,A021,turn_start_walk_0000_002,220713,_1,turn_walk_000_L,figure turns 180 degrees over their left shoulder and then walks forward,a quick turn over the left shoulder followed by a forward walk,character turns 180 degrees over left shoulder and moves forward,A person turns back over their left shoulder and starts walking forward.,"start left foot, turn left 180 degrees over left shoulder, walk forward, move forward","turns back over left shoulder, walks forward",turn walk 000,neutral,neutral,"walking, turning",standing,1,0,0,0,0,A021,S,155,23,129,30,85,121,152,85,24,44,7,41,29,F,general
|
| 56 |
+
jog_ff_start_000_002__A025,jog_backward_start_002__A025,680,Locomotion,Basic Locomotion Neutral,1.0,False,soma_uniform/bvh/220714/jog_backward_start_002__A025.bvh,soma_shapes/soma_base_fit_mhr_params.npz,soma_proportional/bvh/220714/jog_backward_start_002__A025.bvh,soma_shapes/soma_proportion_fit_mhr_params/A025.npz,g1/csv/220714/jog_backward_start_002__A025.csv,jog_ff_start_000_002,A025,jog_backward_start_002,220714,_1,jog_ff_start_000,"start a backwards jog, leading with the left leg","jog backward, starting with the left leg","person performing a backwards jog, facing forward, initiating movement with the left leg",A standing person starts jogging backward steadily by taking medium-length strides while facing forward.,"face forward, jogging back, left leg first, start",begin jog backwards,jog ff start 000,neutral,neutral,jogging,standing,1,1,0,0,0,A025,T,199,31,165,44,103,157,201,113,28,58,10,102,19,M,general
|
| 57 |
+
jump_ff_000_002__A021,jump_backward_002__A021,624,Locomotion,Basic Locomotion Neutral,1.0,False,soma_uniform/bvh/220713/jump_backward_002__A021.bvh,soma_shapes/soma_base_fit_mhr_params.npz,soma_proportional/bvh/220713/jump_backward_002__A021.bvh,soma_shapes/soma_proportion_fit_mhr_params/A021.npz,g1/csv/220713/jump_backward_002__A021.csv,jump_ff_000_002,A021,jump_backward_002,220713,_1,jump_ff_000,"jumping back, landing on the right foot",jumping backwards and landing on the right leg,"a jump backwards led by the left foot, while constantly looking forwards",A person jumps backward and then stands in an upright posture.,"jumping back, left foot first, head looking forward","jump backwards, finish with right leg",jump ff 000,neutral,neutral,jumping,standing,1,1,0,0,0,A021,S,155,23,129,30,85,121,152,85,24,44,7,41,29,F,general
|
| 58 |
+
jog_ff_start_315_R_002__A050,jog_ff_start_315_002__A050,438,Locomotion,Basic Locomotion Neutral,1.0,False,soma_uniform/bvh/221011/jog_ff_start_315_002__A050.bvh,soma_shapes/soma_base_fit_mhr_params.npz,soma_proportional/bvh/221011/jog_ff_start_315_002__A050.bvh,soma_shapes/soma_proportion_fit_mhr_params/A050.npz,g1/csv/221011/jog_ff_start_315_002__A050.csv,jog_ff_start_315_R_002,A050,jog_ff_start_315_002,221011,_1,jog_ff_start_315_R,person starts jogging backward diagonally with the right foot,person jogs backward diagonally starting with the right foot,person starts a diagonal jog backward,A person swiftly jogs backward along the back-right diagonal.,"start right foot, move backward diagonally, left foot back crossing, jog",start jog back right diagonally,jog ff start 315,neutral,neutral,jogging,standing,1,1,0,0,0,A050,T,182,26,152,40,99,140,180,105,24,53,10,75,31,F,general
|
| 59 |
+
turn_jump_090_002__A025,turn_jump_0090_002__A025,387,Locomotion,Basic Locomotion Neutral,1.0,False,soma_uniform/bvh/220714/turn_jump_0090_002__A025.bvh,soma_shapes/soma_base_fit_mhr_params.npz,soma_proportional/bvh/220714/turn_jump_0090_002__A025.bvh,soma_shapes/soma_proportion_fit_mhr_params/A025.npz,g1/csv/220714/turn_jump_0090_002__A025.csv,turn_jump_090_002,A025,turn_jump_0090_002,220714,_1,turn_jump_090,individual performing a leftward turning jump step,a one-step jump with a 90-degree rotation over the left shoulder,individual turns 90 degrees over their left shoulder while jumping one step,A person swiftly pivots and jumps left with agility.,rotate 90 degrees over your left shoulder while jumping one step,turn jump left,turn jump 090,neutral,neutral,"jumping, turning",standing,1,1,0,0,0,A025,T,199,31,165,44,103,157,201,113,28,58,10,102,19,M,general
|
| 60 |
+
crouch_turn_360_R_002__A125,turn_crouch_360_002__A125,433,Locomotion,Advanced Locomotion,1.0,False,soma_uniform/bvh/230119/turn_crouch_360_002__A125.bvh,soma_shapes/soma_base_fit_mhr_params.npz,soma_proportional/bvh/230119/turn_crouch_360_002__A125.bvh,soma_shapes/soma_proportion_fit_mhr_params/A125.npz,g1/csv/230119/turn_crouch_360_002__A125.csv,crouch_turn_360_R_002,A125,turn_crouch_360_002,230119,_3,crouch_turn_360_R,character crouches and turns back over the right shoulder,character makes a 180 degree turn over the right shoulder while crouching,complete a turn back over the right shoulder while keeping a crouch,A person in a crouched position quickly turns back over the right shoulder and then sits facing backward and resting arms on thighs.,"start right foot, turn right 180 degrees over right shoulder, 3 steps, talent crouching, walking in crouch",turning back over right shoulder while crouching,crouch turn 360,neutral,neutral,turning,crouching,0,0,0,0,0,A125,M,175,26,147,30,96,137,180,102,29,53,8,64,28,F,actor
|
| 61 |
+
turn_walk_045_L_002__A023,turn_start_walk_0045_002__A023,423,Locomotion,Basic Locomotion Neutral,1.0,False,soma_uniform/bvh/220713/turn_start_walk_0045_002__A023.bvh,soma_shapes/soma_base_fit_mhr_params.npz,soma_proportional/bvh/220713/turn_start_walk_0045_002__A023.bvh,soma_shapes/soma_proportion_fit_mhr_params/A023.npz,g1/csv/220713/turn_start_walk_0045_002__A023.csv,turn_walk_045_L_002,A023,turn_start_walk_0045_002,220713,_1,turn_walk_045_L,character takes strides forward after a 135-degree left turn over the left shoulder,"character makes a 135 degree left turn, walking forward","person takes a left step, turns left 135 degrees and walks forward",A person turns around towards their left back diagonal and starts walking steadily.,"start left foot, turn left 135 degrees over left shoulder, walk forward diagonally",turns left walks forward,turn walk 045,neutral,neutral,"walking, turning",standing,1,0,0,0,0,A023,T,190,31,158,42,106,160,195,105,27,54,10,90,46,M,general
|
| 62 |
+
jump_twice_R_002__A533,jump_twice_R_002__A533,394,Locomotion,Basic Locomotion Neutral,1.0,False,soma_uniform/bvh/240327/jump_twice_R_002__A533.bvh,soma_shapes/soma_base_fit_mhr_params.npz,soma_proportional/bvh/240327/jump_twice_R_002__A533.bvh,soma_shapes/soma_proportion_fit_mhr_params/A533.npz,g1/csv/240327/jump_twice_R_002__A533.csv,jump_twice_R_002,A533,jump_twice_R_002,240327,_1,jump_twice_R,jump and jump again,person jumps forward twice,two big jumps one after the other,A person energetically jumps twice in a row.,person jumps forward right leg first and then jumps again,jump twice in a row,jump twice,neutral,neutral,jumping,standing,1,1,0,0,1,A533,M,171,29,141,35,90,134,172,95,26,54,11,71,27,M,actor
|
| 63 |
+
relaxed_fast_walk_ff_180_002__A005,walk_fast_forward_relax_002__A005,2178,Locomotion,Basic Locomotion Styles,0.0,False,soma_uniform/bvh/210707/walk_fast_forward_relax_002__A005.bvh,soma_shapes/soma_base_fit_mhr_params.npz,soma_proportional/bvh/210707/walk_fast_forward_relax_002__A005.bvh,soma_shapes/soma_proportion_fit_mhr_params/A005.npz,g1/csv/210707/walk_fast_forward_relax_002__A005.csv,relaxed_fast_walk_ff_180_002,A005,walk_fast_forward_relax_002,210707,_1,relaxed_fast_walk_ff_180,a casual individual briskly walking forward with a sense of calmness,"person walks at a brisk pace forward, maintaining an open and relaxed posture","walk forward quickly, doing so with a relaxed, open posture","A person with arms raised sideways lowers them and walks forward quickly, turning around during the walk and then stands with their arms raised sideways.","person walks forward, high energy, open position, calm",relaxed fast walk facing forward,relaxed fast walk ff 180,relaxed,neutral,walking,standing,1,0,0,0,0,A005,S,152,25,129,32,85,120,153,92,28,45,7,50,23,F,general
|
| 64 |
+
crouch_ff_stop_360_R_002__A126,crouch_ff_stop_360_002__A126,819,Locomotion,Advanced Locomotion,1.0,False,soma_uniform/bvh/230119/crouch_ff_stop_360_002__A126.bvh,soma_shapes/soma_base_fit_mhr_params.npz,soma_proportional/bvh/230119/crouch_ff_stop_360_002__A126.bvh,soma_shapes/soma_proportion_fit_mhr_params/A126.npz,g1/csv/230119/crouch_ff_stop_360_002__A126.csv,crouch_ff_stop_360_R_002,A126,crouch_ff_stop_360_002,230119,_3,crouch_ff_stop_360_R,person stops crouched walk and turns over their right shoulder,"a crouching person, moves backward and stops over the right shoulder",person crouching walks backwards and stops over their right shoulder,The crouched person walks backward while facing forward with arms swaying.,"end left foot, move backward, talent crouching, walking in crouch",stopping crouch back over right shoulder,crouch ff stop 360,neutral,neutral,walking,crouching,1,1,0,0,0,A126,M,169,25,136,25,90,130,164,95,31,51,8,63,25,F,actor
|
| 65 |
+
high_jump_full_turn_R_opt_1_002__A477,high_jump_full_turn_R_opt_1_002__A477,427,Locomotion,Advanced Locomotion,1.0,False,soma_uniform/bvh/231011/high_jump_full_turn_R_opt_1_002__A477.bvh,soma_shapes/soma_base_fit_mhr_params.npz,soma_proportional/bvh/231011/high_jump_full_turn_R_opt_1_002__A477.bvh,soma_shapes/soma_proportion_fit_mhr_params/A477.npz,g1/csv/231011/high_jump_full_turn_R_opt_1_002__A477.csv,high_jump_full_turn_R_opt_1_002,A477,high_jump_full_turn_R_opt_1_002,231011,_1,high_jump_full_turn_R_opt_1,turning around during high jump up,the performer turns around while jumping high up,the performer turns around during a high jump,"A person quickly jumps while rotating right, turns around mid-air, and lands on the ground facing forward.",character turns around during a high jump up,turning around during a high jump up and landing,high jump full turn opt 1,neutral,neutral,jumping,standing,0,1,0,0,0,A477,M,168,28,140,28,97,132,170,98,29,52,11,62,28,F,actor
|
| 66 |
+
jump_ff_360_R_003__A045,jump_ff_360_003__A045,655,Locomotion,Baseline,1.0,False,soma_uniform/bvh/221010/jump_ff_360_003__A045.bvh,soma_shapes/soma_base_fit_mhr_params.npz,soma_proportional/bvh/221010/jump_ff_360_003__A045.bvh,soma_shapes/soma_proportion_fit_mhr_params/A045.npz,g1/csv/221010/jump_ff_360_003__A045.csv,jump_ff_360_R_003,A045,jump_ff_360_003,221010,_1,jump_ff_360_R,character performs a small jump while moving backward,"begin by moving the right foot, then jump backward",person takes a small jump backward,A person jumps backward while facing forward.,"start right foot, move backward, small jump",jump backwards,jump ff 360,neutral,neutral,jumping,standing,1,1,0,0,0,A045,S,156,25,130,39,82,123,154,87,28,45,8,59,22,F,general
|
| 67 |
+
crouch_idle_003__A148,idle_crouch_loop_003__A148,831,Locomotion,Advanced Locomotion,1.0,False,soma_uniform/bvh/230125/idle_crouch_loop_003__A148.bvh,soma_shapes/soma_base_fit_mhr_params.npz,soma_proportional/bvh/230125/idle_crouch_loop_003__A148.bvh,soma_shapes/soma_proportion_fit_mhr_params/A148.npz,g1/csv/230125/idle_crouch_loop_003__A148.csv,crouch_idle_003,A148,idle_crouch_loop_003,230125,_2,crouch_idle,assume a crouching position with feet almost parallel,assumes a squatting posture with nearly parallel feet in an open position,person is crouching down with their feet in a parallel and open position,A person is sitting in a crouched posture.,"feet almost parallel in open position, talent crouching",crouch position,crouch idle,neutral,neutral,walking,crouching,0,0,0,0,0,A148,M,181,29,152,35,99,145,188,103,29,58,10,73,25,M,actor
|
| 68 |
+
high_jump_R_opt_1_003__A280,high_jump_R_003__A280,514,Locomotion,Baseline,1.0,False,soma_uniform/bvh/230316/high_jump_R_003__A280.bvh,soma_shapes/soma_base_fit_mhr_params.npz,soma_proportional/bvh/230316/high_jump_R_003__A280.bvh,soma_shapes/soma_proportion_fit_mhr_params/A280.npz,g1/csv/230316/high_jump_R_003__A280.csv,high_jump_R_opt_1_003,A280,high_jump_R_003,230316,_1,high_jump_R_opt_1,high jump over a moving pole in one place,making high jump over a moving pole stationary,performer makes high jump in one place over a moving pole,A person energetically does a high jump while facing forward.,"jump high over moving pole, jump high in one place",jumping high,high jump opt 1,neutral,neutral,jumping,standing,1,1,0,0,0,A280,M,171,27,141,32,93,134,170,99,27,50,9,68,25,M,actor
|
| 69 |
+
crouch_idle_right_R_003__A194,idle_crouch_right_loop_R_003__A194,1225,Locomotion,Advanced Locomotion,1.0,False,soma_uniform/bvh/230215/idle_crouch_right_loop_R_003__A194.bvh,soma_shapes/soma_base_fit_mhr_params.npz,soma_proportional/bvh/230215/idle_crouch_right_loop_R_003__A194.bvh,soma_shapes/soma_proportion_fit_mhr_params/A194.npz,g1/csv/230215/idle_crouch_right_loop_R_003__A194.csv,crouch_idle_right_R_003,A194,idle_crouch_right_loop_R_003,230215,_1,crouch_idle_right_R,character with right foot forward assumes a crouching position leaning on the left foot,stand in a crouching position with right foot forward,maintain a crouching position with the right leg slightly extended forward,"A person maintains a crouched position with their right leg forward, making slight movements with their forearms.","right foot slightly forward, body weight on left foot, talent crouching",crouch position right leg forward,crouch idle right,neutral,neutral,crouching idle,crouching,0,0,0,0,0,A194,M,178,29,146,29,91,138,173,98,32,53,10,90,25,M,actor
|
| 70 |
+
walk_ff_start_315_R_003__A049,walk_ff_start_315_003__A049,800,Locomotion,Basic Locomotion Neutral,1.0,False,soma_uniform/bvh/221011/walk_ff_start_315_003__A049.bvh,soma_shapes/soma_base_fit_mhr_params.npz,soma_proportional/bvh/221011/walk_ff_start_315_003__A049.bvh,soma_shapes/soma_proportion_fit_mhr_params/A049.npz,g1/csv/221011/walk_ff_start_315_003__A049.csv,walk_ff_start_315_R_003,A049,walk_ff_start_315_003,221011,_1,walk_ff_start_315_R,walking backward starting with right foot and moving diagonally with left foot crossing behind,"begin by moving right foot, then move backward diagonally crossing with left foot",person takes a right diagonal step backward,A person starts walking swiftly toward the back right diagonal in a crosswise manner at a moderate pace.,"start right foot, move backward diagonally, left foot back crossing, walk backward diagonally",start walking back right diagonal,walk ff start 315,neutral,neutral,walking,standing,1,0,0,0,0,A049,S,156,23,130,40,84,120,154,88,24,47,9,50,25,F,general
|
| 71 |
+
jog_arc_ccw_stop_003__A029,arc_jog_left_stop_003__A029,533,Locomotion,Basic Locomotion Neutral,1.0,False,soma_uniform/bvh/220720/arc_jog_left_stop_003__A029.bvh,soma_shapes/soma_base_fit_mhr_params.npz,soma_proportional/bvh/220720/arc_jog_left_stop_003__A029.bvh,soma_shapes/soma_proportion_fit_mhr_params/A029.npz,g1/csv/220720/arc_jog_left_stop_003__A029.csv,jog_arc_ccw_stop_003,A029,arc_jog_left_stop_003,220720,_1,jog_arc_ccw_stop,"person jogs in a counterclockwise circle, then abruptly stops",person stops jogging in a counterclockwise arc,stop jogging in a counterclockwise circle,A person jogs actively towards left in a semicircle.,"jogging in a circle, counterclockwise, stop",stop jogging counterclockwise in an arc,jog arc ccw stop,neutral,neutral,jogging,standing,1,1,0,0,0,A029,S,155,24,127,31,85,120,158,80,22,43,8,41,22,F,general
|
| 72 |
+
walk_ff_stop_360_R_003__A049,walk_ff_stop_360_003__A049,198,Locomotion,Basic Locomotion Neutral,1.0,False,soma_uniform/bvh/221011/walk_ff_stop_360_003__A049.bvh,soma_shapes/soma_base_fit_mhr_params.npz,soma_proportional/bvh/221011/walk_ff_stop_360_003__A049.bvh,soma_shapes/soma_proportion_fit_mhr_params/A049.npz,g1/csv/221011/walk_ff_stop_360_003__A049.csv,walk_ff_stop_360_R_003,A049,walk_ff_stop_360_003,221011,_1,walk_ff_stop_360_R,person stops and walks backwards,stop the action of walking backwards,"individual ends their backward walk, starting with their left foot",The person stops walking backwards.,"end left foot, move backward, walk backward",stop walking backwards,walk ff stop 360,neutral,neutral,walking,standing,1,0,0,0,0,A049,S,156,23,130,40,84,120,154,88,24,47,9,50,25,F,general
|
| 73 |
+
crouch_ff_stop_180_R_003__A146,crouch_ff_stop_180_R_003__A146,674,Locomotion,Advanced Locomotion,1.0,False,soma_uniform/bvh/230125/crouch_ff_stop_180_R_003__A146.bvh,soma_shapes/soma_base_fit_mhr_params.npz,soma_proportional/bvh/230125/crouch_ff_stop_180_R_003__A146.bvh,soma_shapes/soma_proportion_fit_mhr_params/A146.npz,g1/csv/230125/crouch_ff_stop_180_R_003__A146.csv,crouch_ff_stop_180_R_003,A146,crouch_ff_stop_180_R_003,230125,_2,crouch_ff_stop_180_R,walking in a forwards direction while in a crouching position,a crouching character stops and faces forward after moving forward,start walking forward while maintaining a crouch position,A crouched person walks forward and then stands for a while.,"end left foot, move forward, talent crouching, walking in crouch",stopping crouch facing forward,crouch ff stop 180,neutral,neutral,walking,crouching,1,1,0,0,0,A146,M,168,27,140,30,95,138,171,90,26,52,8,57,25,F,actor
|
| 74 |
+
turn_jump_000_003__A022,turn_jump_0000_003__A022,465,Locomotion,Basic Locomotion Neutral,1.0,False,soma_uniform/bvh/220713/turn_jump_0000_003__A022.bvh,soma_shapes/soma_base_fit_mhr_params.npz,soma_proportional/bvh/220713/turn_jump_0000_003__A022.bvh,soma_shapes/soma_proportion_fit_mhr_params/A022.npz,g1/csv/220713/turn_jump_0000_003__A022.csv,turn_jump_000_003,A022,turn_jump_0000_003,220713,_1,turn_jump_000,jumping back over the left shoulder with a full spin,a jump over the left shoulder with a 180-degree turn,person leaps one step while rotating 180 degrees over the left shoulder,A person quickly jumps back over their left shoulder and then stands.,rotate 180 degrees over your left shoulder while jumping one step,turn jump back over left shoulder,turn jump 000,neutral,neutral,"jumping, turning",standing,1,1,0,0,0,A022,T,180,25,152,32,93,135,174,105,28,51,7,62,30,F,general
|
| 75 |
+
jog_ff_start_180_R_003__A022,jog_forward_start_003__A022,424,Locomotion,Baseline,1.0,False,soma_uniform/bvh/220713/jog_forward_start_003__A022.bvh,soma_shapes/soma_base_fit_mhr_params.npz,soma_proportional/bvh/220713/jog_forward_start_003__A022.bvh,soma_shapes/soma_proportion_fit_mhr_params/A022.npz,g1/csv/220713/jog_forward_start_003__A022.csv,jog_ff_start_180_R_003,A022,jog_forward_start_003,220713,_1,jog_ff_start_180_R,start jogging forward,start a forward jog,initiate a forward jog,A person standing on the ground steadily jogs in the forward direction and then stands on the ground in an upright position.,"start right foot, move forward, jog",start jog forward,jog ff start 180,neutral,neutral,jogging,standing,1,1,0,0,0,A022,T,180,25,152,32,93,135,174,105,28,51,7,62,30,F,general
|
| 76 |
+
jog_arc_ccw_start_003__A029,arc_jog_left_start_003__A029,512,Locomotion,Basic Locomotion Neutral,1.0,False,soma_uniform/bvh/220720/arc_jog_left_start_003__A029.bvh,soma_shapes/soma_base_fit_mhr_params.npz,soma_proportional/bvh/220720/arc_jog_left_start_003__A029.bvh,soma_shapes/soma_proportion_fit_mhr_params/A029.npz,g1/csv/220720/arc_jog_left_start_003__A029.csv,jog_arc_ccw_start_003,A029,arc_jog_left_start_003,220720,_1,jog_arc_ccw_start,jogging is initiated in a counterclockwise arc,commence a jog in counterclockwise direction,person starts jogging in a counterclockwise arc,A person standing starts jogging in an anticlockwise arc.,"jogging in a circle, counterclockwise, start",start jogging counterclockwise in an arc,jog arc ccw start,neutral,neutral,jogging,standing,1,1,0,0,0,A029,S,155,24,127,31,85,120,158,80,22,43,8,41,22,F,general
|
| 77 |
+
jog_arc_cw_stop_R_003__A047,jog_arc_cw_stop_003__A047,821,Locomotion,Basic Locomotion Neutral,1.0,False,soma_uniform/bvh/221010/jog_arc_cw_stop_003__A047.bvh,soma_shapes/soma_base_fit_mhr_params.npz,soma_proportional/bvh/221010/jog_arc_cw_stop_003__A047.bvh,soma_shapes/soma_proportion_fit_mhr_params/A047.npz,g1/csv/221010/jog_arc_cw_stop_003__A047.csv,jog_arc_cw_stop_R_003,A047,jog_arc_cw_stop_003,221010,_2,jog_arc_cw_stop_R,"someone stops jogging in a clockwise, arc-like motion",stop and jog clockwise forming a circular path,person finishes jogging in a clockwise arc,"A person is jogging in a clockwise arc over the right shoulder, then stops and stands upright.","end left foot, move forward in a circle, clockwise, jog ",stop jogging clockwise in an arc,jog arc cw stop,neutral,neutral,jogging,standing,1,1,0,0,0,A047,T,189,29,156,48,103,153,193,103,29,58,10,100,33,M,general
|
| 78 |
+
walk_ff_stop_180_R_003__A025,walk_forward_stop_003__A025,907,Locomotion,Basic Locomotion Neutral,1.0,False,soma_uniform/bvh/220714/walk_forward_stop_003__A025.bvh,soma_shapes/soma_base_fit_mhr_params.npz,soma_proportional/bvh/220714/walk_forward_stop_003__A025.bvh,soma_shapes/soma_proportion_fit_mhr_params/A025.npz,g1/csv/220714/walk_forward_stop_003__A025.csv,walk_ff_stop_180_R_003,A025,walk_forward_stop_003,220714,_1,walk_ff_stop_180_R,stop moving forward by landing on the left foot while walking,"advance forward and stop with a flourish, landing on your left foot","finish walking forward, ending with the left foot",A person standing is walking in the forward direction at a steady pace.,"end left foot, move forward, walk forward, maximum expression","stop walking forward, finish with left leg",walk ff stop 180,neutral,neutral,walking,standing,1,0,0,0,0,A025,T,199,31,165,44,103,157,201,113,28,58,10,102,19,M,general
|
| 79 |
+
turn_walk_000_L_003__A021,turn_start_walk_0000_003__A021,411,Locomotion,Basic Locomotion Neutral,1.0,False,soma_uniform/bvh/220713/turn_start_walk_0000_003__A021.bvh,soma_shapes/soma_base_fit_mhr_params.npz,soma_proportional/bvh/220713/turn_start_walk_0000_003__A021.bvh,soma_shapes/soma_proportion_fit_mhr_params/A021.npz,g1/csv/220713/turn_start_walk_0000_003__A021.csv,turn_walk_000_L_003,A021,turn_start_walk_0000_003,220713,_1,turn_walk_000_L,figure turns 180 degrees over their left shoulder and then walks forward,a quick turn over the left shoulder followed by a forward walk,character turns 180 degrees over left shoulder and moves forward,A person turns back over their left shoulder and starts walking forward.,"start left foot, turn left 180 degrees over left shoulder, walk forward, move forward","turns back over left shoulder, walks forward",turn walk 000,neutral,neutral,"walking, turning",standing,1,0,0,0,0,A021,S,155,23,129,30,85,121,152,85,24,44,7,41,29,F,general
|
| 80 |
+
jog_ff_start_000_003__A025,jog_backward_start_003__A025,666,Locomotion,Basic Locomotion Neutral,1.0,False,soma_uniform/bvh/220714/jog_backward_start_003__A025.bvh,soma_shapes/soma_base_fit_mhr_params.npz,soma_proportional/bvh/220714/jog_backward_start_003__A025.bvh,soma_shapes/soma_proportion_fit_mhr_params/A025.npz,g1/csv/220714/jog_backward_start_003__A025.csv,jog_ff_start_000_003,A025,jog_backward_start_003,220714,_1,jog_ff_start_000,"start a backwards jog, leading with the left leg","jog backward, starting with the left leg","person performing a backwards jog, facing forward, initiating movement with the left leg",A standing person starts jogging backward steadily by taking medium-length strides while facing forward.,"face forward, jogging back, left leg first, start",begin jog backwards,jog ff start 000,neutral,neutral,jogging,standing,1,1,0,0,0,A025,T,199,31,165,44,103,157,201,113,28,58,10,102,19,M,general
|
| 81 |
+
jump_ff_000_003__A021,jump_backward_003__A021,284,Locomotion,Basic Locomotion Neutral,1.0,False,soma_uniform/bvh/220713/jump_backward_003__A021.bvh,soma_shapes/soma_base_fit_mhr_params.npz,soma_proportional/bvh/220713/jump_backward_003__A021.bvh,soma_shapes/soma_proportion_fit_mhr_params/A021.npz,g1/csv/220713/jump_backward_003__A021.csv,jump_ff_000_003,A021,jump_backward_003,220713,_1,jump_ff_000,"jumping back, landing on the right foot",jumping backwards and landing on the right leg,"a jump backwards led by the left foot, while constantly looking forwards",A person jumps backward and then stands in an upright posture.,"jumping back, left foot first, head looking forward","jump backwards, finish with right leg",jump ff 000,neutral,neutral,jumping,standing,1,1,0,0,0,A021,S,155,23,129,30,85,121,152,85,24,44,7,41,29,F,general
|
| 82 |
+
jog_ff_start_315_R_003__A050,jog_ff_start_315_003__A050,438,Locomotion,Basic Locomotion Neutral,1.0,False,soma_uniform/bvh/221011/jog_ff_start_315_003__A050.bvh,soma_shapes/soma_base_fit_mhr_params.npz,soma_proportional/bvh/221011/jog_ff_start_315_003__A050.bvh,soma_shapes/soma_proportion_fit_mhr_params/A050.npz,g1/csv/221011/jog_ff_start_315_003__A050.csv,jog_ff_start_315_R_003,A050,jog_ff_start_315_003,221011,_1,jog_ff_start_315_R,person starts jogging backward diagonally with the right foot,person jogs backward diagonally starting with the right foot,person starts a diagonal jog backward,A person swiftly jogs backward along the back-right diagonal.,"start right foot, move backward diagonally, left foot back crossing, jog",start jog back right diagonally,jog ff start 315,neutral,neutral,jogging,standing,1,1,0,0,0,A050,T,182,26,152,40,99,140,180,105,24,53,10,75,31,F,general
|
| 83 |
+
turn_jump_090_003__A023,turn_jump_0090_003__A023,735,Locomotion,Basic Locomotion Neutral,1.0,False,soma_uniform/bvh/220713/turn_jump_0090_003__A023.bvh,soma_shapes/soma_base_fit_mhr_params.npz,soma_proportional/bvh/220713/turn_jump_0090_003__A023.bvh,soma_shapes/soma_proportion_fit_mhr_params/A023.npz,g1/csv/220713/turn_jump_0090_003__A023.csv,turn_jump_090_003,A023,turn_jump_0090_003,220713,_1,turn_jump_090,individual performing a leftward turning jump step,a one-step jump with a 90-degree rotation over the left shoulder,individual turns 90 degrees over their left shoulder while jumping one step,A person swiftly pivots and jumps left with agility.,rotate 90 degrees over your left shoulder while jumping one step,turn jump left,turn jump 090,neutral,neutral,"jumping, turning",standing,1,1,0,0,0,A023,T,190,31,158,42,106,160,195,105,27,54,10,90,46,M,general
|
| 84 |
+
crouch_turn_360_R_003__A125,turn_crouch_360_003__A125,533,Locomotion,Advanced Locomotion,1.0,False,soma_uniform/bvh/230119/turn_crouch_360_003__A125.bvh,soma_shapes/soma_base_fit_mhr_params.npz,soma_proportional/bvh/230119/turn_crouch_360_003__A125.bvh,soma_shapes/soma_proportion_fit_mhr_params/A125.npz,g1/csv/230119/turn_crouch_360_003__A125.csv,crouch_turn_360_R_003,A125,turn_crouch_360_003,230119,_3,crouch_turn_360_R,character crouches and turns back over the right shoulder,character makes a 180 degree turn over the right shoulder while crouching,complete a turn back over the right shoulder while keeping a crouch,A person in a crouched position quickly turns back over the right shoulder and then sits facing backward and resting arms on thighs.,"start right foot, turn right 180 degrees over right shoulder, 3 steps, talent crouching, walking in crouch",turning back over right shoulder while crouching,crouch turn 360,neutral,neutral,turning,crouching,0,0,0,0,0,A125,M,175,26,147,30,96,137,180,102,29,53,8,64,28,F,actor
|
| 85 |
+
turn_walk_045_L_003__A023,turn_start_walk_0045_003__A023,516,Locomotion,Basic Locomotion Neutral,1.0,False,soma_uniform/bvh/220713/turn_start_walk_0045_003__A023.bvh,soma_shapes/soma_base_fit_mhr_params.npz,soma_proportional/bvh/220713/turn_start_walk_0045_003__A023.bvh,soma_shapes/soma_proportion_fit_mhr_params/A023.npz,g1/csv/220713/turn_start_walk_0045_003__A023.csv,turn_walk_045_L_003,A023,turn_start_walk_0045_003,220713,_1,turn_walk_045_L,character takes strides forward after a 135-degree left turn over the left shoulder,"character makes a 135 degree left turn, walking forward","person takes a left step, turns left 135 degrees and walks forward",A person turns around towards their left back diagonal and starts walking steadily.,"start left foot, turn left 135 degrees over left shoulder, walk forward diagonally",turns left walks forward,turn walk 045,neutral,neutral,"walking, turning",standing,1,0,0,0,0,A023,T,190,31,158,42,106,160,195,105,27,54,10,90,46,M,general
|
| 86 |
+
jump_twice_R_003__A533,jump_twice_R_003__A533,483,Locomotion,Basic Locomotion Neutral,1.0,False,soma_uniform/bvh/240327/jump_twice_R_003__A533.bvh,soma_shapes/soma_base_fit_mhr_params.npz,soma_proportional/bvh/240327/jump_twice_R_003__A533.bvh,soma_shapes/soma_proportion_fit_mhr_params/A533.npz,g1/csv/240327/jump_twice_R_003__A533.csv,jump_twice_R_003,A533,jump_twice_R_003,240327,_1,jump_twice_R,jump and jump again,person jumps forward twice,two big jumps one after the other,"A person jumps forward twice in a row, and stands in an upright posture, taking a slight pause between the jumps.",person jumps forward right leg first and then jumps again,jump twice in a row,jump twice,neutral,neutral,jumping,standing,1,1,0,0,1,A533,M,171,29,141,35,90,134,172,95,26,54,11,71,27,M,actor
|
| 87 |
+
relaxed_fast_walk_ff_180_003__A005,walk_fast_forward_relax_003__A005,2252,Locomotion,Basic Locomotion Styles,0.0,False,soma_uniform/bvh/210707/walk_fast_forward_relax_003__A005.bvh,soma_shapes/soma_base_fit_mhr_params.npz,soma_proportional/bvh/210707/walk_fast_forward_relax_003__A005.bvh,soma_shapes/soma_proportion_fit_mhr_params/A005.npz,g1/csv/210707/walk_fast_forward_relax_003__A005.csv,relaxed_fast_walk_ff_180_003,A005,walk_fast_forward_relax_003,210707,_1,relaxed_fast_walk_ff_180,a casual individual briskly walking forward with a sense of calmness,"person walks at a brisk pace forward, maintaining an open and relaxed posture","walk forward quickly, doing so with a relaxed, open posture","A person with arms raised sideways lowers them and walks forward quickly, turning around during the walk and then stands with their arms raised sideways.","person walks forward, high energy, open position, calm",relaxed fast walk facing forward,relaxed fast walk ff 180,relaxed,neutral,walking,standing,1,0,0,0,0,A005,S,152,25,129,32,85,120,153,92,28,45,7,50,23,F,general
|
| 88 |
+
crouch_ff_stop_360_R_003__A126,crouch_ff_stop_360_003__A126,893,Locomotion,Advanced Locomotion,1.0,False,soma_uniform/bvh/230119/crouch_ff_stop_360_003__A126.bvh,soma_shapes/soma_base_fit_mhr_params.npz,soma_proportional/bvh/230119/crouch_ff_stop_360_003__A126.bvh,soma_shapes/soma_proportion_fit_mhr_params/A126.npz,g1/csv/230119/crouch_ff_stop_360_003__A126.csv,crouch_ff_stop_360_R_003,A126,crouch_ff_stop_360_003,230119,_3,crouch_ff_stop_360_R,person stops crouched walk and turns over their right shoulder,"a crouching person, moves backward and stops over the right shoulder",person crouching walks backwards and stops over their right shoulder,The crouched person walks backward while facing forward with arms swaying.,"end left foot, move backward, talent crouching, walking in crouch",stopping crouch back over right shoulder,crouch ff stop 360,neutral,neutral,walking,crouching,1,1,0,0,0,A126,M,169,25,136,25,90,130,164,95,31,51,8,63,25,F,actor
|
| 89 |
+
jump_ff_360_R_004__A045,jump_ff_360_004__A045,665,Locomotion,Baseline,1.0,False,soma_uniform/bvh/221010/jump_ff_360_004__A045.bvh,soma_shapes/soma_base_fit_mhr_params.npz,soma_proportional/bvh/221010/jump_ff_360_004__A045.bvh,soma_shapes/soma_proportion_fit_mhr_params/A045.npz,g1/csv/221010/jump_ff_360_004__A045.csv,jump_ff_360_R_004,A045,jump_ff_360_004,221010,_1,jump_ff_360_R,character performs a small jump while moving backward,"begin by moving the right foot, then jump backward",person takes a small jump backward,A person jumps backward while facing forward.,"start right foot, move backward, small jump",jump backwards,jump ff 360,neutral,neutral,jumping,standing,1,1,0,0,0,A045,S,156,25,130,39,82,123,154,87,28,45,8,59,22,F,general
|
| 90 |
+
crouch_idle_004__A244,crouch_idle_004__A244,975,Locomotion,Advanced Locomotion,1.0,False,soma_uniform/bvh/230306/crouch_idle_004__A244.bvh,soma_shapes/soma_base_fit_mhr_params.npz,soma_proportional/bvh/230306/crouch_idle_004__A244.bvh,soma_shapes/soma_proportion_fit_mhr_params/A244.npz,g1/csv/230306/crouch_idle_004__A244.csv,crouch_idle_004,A244,crouch_idle_004,230306,_1,crouch_idle,assume a crouching position with feet almost parallel,assumes a squatting posture with nearly parallel feet in an open position,person is crouching down with their feet in a parallel and open position,A crouched person is standing in place and subtly swaying.,"feet almost parallel in open position, talent crouching",crouch position,crouch idle,neutral,neutral,walking,crouching,0,0,0,0,0,A244,M,169,28,140,29,95,137,173,95,26,50,8,61,27,F,actor
|
| 91 |
+
high_jump_R_opt_1_004__A305,high_jump_R_004__A305,413,Locomotion,Baseline,1.0,False,soma_uniform/bvh/230324/high_jump_R_004__A305.bvh,soma_shapes/soma_base_fit_mhr_params.npz,soma_proportional/bvh/230324/high_jump_R_004__A305.bvh,soma_shapes/soma_proportion_fit_mhr_params/A305.npz,g1/csv/230324/high_jump_R_004__A305.csv,high_jump_R_opt_1_004,A305,high_jump_R_004,230324,_1,high_jump_R_opt_1,high jump over a moving pole in one place,making high jump over a moving pole stationary,performer makes high jump in one place over a moving pole,A person energetically does a high jump while facing forward.,"jump high over moving pole, jump high in one place",jumping high,high jump opt 1,neutral,neutral,jumping,standing,1,1,0,0,0,A305,M,167,25,141,36,87,127,164,97,28,53,10,56,30,F,stuntman
|
| 92 |
+
crouch_idle_right_R_101__A125,idle_crouch_right_loop_101__A125,911,Locomotion,Advanced Locomotion,1.0,False,soma_uniform/bvh/230119/idle_crouch_right_loop_101__A125.bvh,soma_shapes/soma_base_fit_mhr_params.npz,soma_proportional/bvh/230119/idle_crouch_right_loop_101__A125.bvh,soma_shapes/soma_proportion_fit_mhr_params/A125.npz,g1/csv/230119/idle_crouch_right_loop_101__A125.csv,crouch_idle_right_R_101,A125,idle_crouch_right_loop_101,230119,_3,crouch_idle_right_R,character with right foot forward assumes a crouching position leaning on the left foot,stand in a crouching position with right foot forward,maintain a crouching position with the right leg slightly extended forward,A crouched person is standing in place with their right leg ahead of the left.,"right foot slightly forward, body weight on left foot, talent crouching",crouch position right leg forward,crouch idle right,neutral,neutral,crouching idle,crouching,0,0,0,0,0,A125,M,175,26,147,30,96,137,180,102,29,53,8,64,28,F,actor
|
| 93 |
+
walk_ff_start_315_R_101__A174,walk_ff_start_315_101__A174,572,Locomotion,Basic Locomotion Neutral,1.0,False,soma_uniform/bvh/230209/walk_ff_start_315_101__A174.bvh,soma_shapes/soma_base_fit_mhr_params.npz,soma_proportional/bvh/230209/walk_ff_start_315_101__A174.bvh,soma_shapes/soma_proportion_fit_mhr_params/A174.npz,g1/csv/230209/walk_ff_start_315_101__A174.csv,walk_ff_start_315_R_101,A174,walk_ff_start_315_101,230209,_1,walk_ff_start_315_R,walking backward starting with right foot and moving diagonally with left foot crossing behind,"begin by moving right foot, then move backward diagonally crossing with left foot",person takes a right diagonal step backward,A person starts walking swiftly toward the back right diagonal in a crosswise manner at a moderate pace.,"start right foot, move backward diagonally, left foot back crossing, walk backward diagonally",start walking back right diagonal,walk ff start 315,neutral,neutral,walking,standing,1,0,0,0,0,A174,M,169,23,140,29,87,130,163,98,24,51,8,56,31,F,actor
|
| 94 |
+
kneeling_start_R_001__A021,kneeling_start_001__A021,572,Locomotion,Basic Locomotion Neutral,1.0,False,soma_uniform/bvh/220713/kneeling_start_001__A021.bvh,soma_shapes/soma_base_fit_mhr_params.npz,soma_proportional/bvh/220713/kneeling_start_001__A021.bvh,soma_shapes/soma_proportion_fit_mhr_params/A021.npz,g1/csv/220713/kneeling_start_001__A021.csv,kneeling_start_R_001,A021,kneeling_start_001,220713,_1,kneeling_start_R,"person begins to kneel, leading with the right knee","person starts to kneel, putting their right knee down first",person starts to kneel with their right knee first and keeps a straight back,"A person standing upright kneels down while facing forward, and sways slightly.","from standing to kneeling, right knee first, back straight while kneeling",beginning to kneel,kneeling start,neutral,neutral,kneeling,"standing, kneeling",0,0,0,0,0,A021,S,155,23,129,30,85,121,152,85,24,44,7,41,29,F,general
|
| 95 |
+
kneeling_stop_R_001__A021,kneeling_stop_001__A021,389,Locomotion,Basic Locomotion Neutral,1.0,False,soma_uniform/bvh/220713/kneeling_stop_001__A021.bvh,soma_shapes/soma_base_fit_mhr_params.npz,soma_proportional/bvh/220713/kneeling_stop_001__A021.bvh,soma_shapes/soma_proportion_fit_mhr_params/A021.npz,g1/csv/220713/kneeling_stop_001__A021.csv,kneeling_stop_R_001,A021,kneeling_stop_001,220713,_1,kneeling_stop_R,"transition from a kneeling to standing position, leading with the right foot and keeping the back straight","person transitions from kneeling to standing, leading with the right foot and keeping their back straight","individual transitions from a kneeling position to standing upright, leading with their right foot and keeping their back straight",A person kneeling on the ground gets up and stands idle.,"from kneeling to standing, right foot first, back straight while kneeling",stopping kneeling,kneeling stop,neutral,neutral,kneeling,"standing, kneeling",1,1,0,0,0,A021,S,155,23,129,30,85,121,152,85,24,44,7,41,29,F,general
|
| 96 |
+
walk_ff_start_180_R_001__A021,walk_forward_start_001__A021,594,Locomotion,Basic Locomotion Neutral,1.0,False,soma_uniform/bvh/220713/walk_forward_start_001__A021.bvh,soma_shapes/soma_base_fit_mhr_params.npz,soma_proportional/bvh/220713/walk_forward_start_001__A021.bvh,soma_shapes/soma_proportion_fit_mhr_params/A021.npz,g1/csv/220713/walk_forward_start_001__A021.csv,walk_ff_start_180_R_001,A021,walk_forward_start_001,220713,_1,walk_ff_start_180_R,"starting with the right leg, person takes a step forward",initiate a walk forward by stepping out with your right foot,start walking forward beginning with the right foot,A person starts to walk forward at a normal pace.,"start with the right foot, move forward, and walk forward",beginning to walk forward with the right leg,walk ff start 180,neutral,neutral,walking,standing,1,0,0,0,0,A021,S,155,23,129,30,85,121,152,85,24,44,7,41,29,F,general
|
| 97 |
+
walk_ff_loop_270_R_001__A021,walk_sideway_right_loop_001__A021,1801,Locomotion,Basic Locomotion Neutral,1.0,False,soma_uniform/bvh/220713/walk_sideway_right_loop_001__A021.bvh,soma_shapes/soma_base_fit_mhr_params.npz,soma_proportional/bvh/220713/walk_sideway_right_loop_001__A021.bvh,soma_shapes/soma_proportion_fit_mhr_params/A021.npz,g1/csv/220713/walk_sideway_right_loop_001__A021.csv,walk_ff_loop_270_R_001,A021,walk_sideway_right_loop_001,220713,_1,walk_ff_loop_270_R,tired person is taking side steps to the right,tired person walks sideways to the right with lead foot first,tired person walks sideways to the right,"A person is steadily walking sideways in the right direction, taking several steps.","face forward all the time, lead foot first, passing step, walk right, talent tired all the time, low energy",walking sideways to the right,walk ff loop 270,neutral,neutral,walking,standing,1,0,0,0,0,A021,S,155,23,129,30,85,121,152,85,24,44,7,41,29,F,general
|
| 98 |
+
jog_ff_loop_180_R_001__A017,Loop_Forward_Jog_001__A017,14153,Locomotion,Baseline,1.0,False,soma_uniform/bvh/220705/Loop_Forward_Jog_001__A017.bvh,soma_shapes/soma_base_fit_mhr_params.npz,soma_proportional/bvh/220705/Loop_Forward_Jog_001__A017.bvh,soma_shapes/soma_proportion_fit_mhr_params/A017.npz,g1/csv/220705/Loop_Forward_Jog_001__A017.csv,jog_ff_loop_180_R_001,A017,Loop_Forward_Jog_001,220705,_1,jog_ff_loop_180_R,jog forward ,person jogs forward without speeding up to a run,character is progressing in a jogging pace,"A person jogs forward, comes to a halt, stands still, and then tilts to the right.","move forward, jog",jog forward,jog ff loop 180,neutral,neutral,jogging,standing,1,1,0,0,0,A017,T,180,27,148,43,95,144,178,100,27,50,9,60,32,F,general
|
| 99 |
+
walk_ff_loop_180_R_001__A017,Loop_Forward_Walk_001__A017,1013,Locomotion,Basic Locomotion Neutral,1.0,False,soma_uniform/bvh/220705/Loop_Forward_Walk_001__A017.bvh,soma_shapes/soma_base_fit_mhr_params.npz,soma_proportional/bvh/220705/Loop_Forward_Walk_001__A017.bvh,soma_shapes/soma_proportion_fit_mhr_params/A017.npz,g1/csv/220705/Loop_Forward_Walk_001__A017.csv,walk_ff_loop_180_R_001,A017,Loop_Forward_Walk_001,220705,_1,walk_ff_loop_180_R,keep walking forward,character walks forward keeping their face looking front,"someone walks forward, leading with their right foot",A person casually strolls forward at a normal pace.,"consistently face forward, start with the right foot and walk forward",strolling forward,walk ff loop 180,neutral,neutral,walking,standing,1,0,0,0,0,A017,T,180,27,148,43,95,144,178,100,27,50,9,60,32,F,general
|
| 100 |
+
jump_ff_180_R_001__A021,jump_forward_001__A021,1144,Locomotion,Baseline,1.0,False,soma_uniform/bvh/220713/jump_forward_001__A021.bvh,soma_shapes/soma_base_fit_mhr_params.npz,soma_proportional/bvh/220713/jump_forward_001__A021.bvh,soma_shapes/soma_proportion_fit_mhr_params/A021.npz,g1/csv/220713/jump_forward_001__A021.csv,jump_ff_180_R_001,A021,jump_forward_001,220713,_1,jump_ff_180_R,jumping forward from a standing position,make a small forward leap,person makes a small jump as they move forward,"A person jumps forward, pauses briefly in a standing position, then begins to walk backward while turning to their right.","start right foot, move forward, small jump",jump forward,jump ff 180,neutral,neutral,jumping,standing,1,1,0,0,0,A021,S,155,23,129,30,85,121,152,85,24,44,7,41,29,F,general
|
| 101 |
+
jump_ff_180_R_001__A022,jump_forward_001__A022,1260,Locomotion,Baseline,1.0,False,soma_uniform/bvh/220713/jump_forward_001__A022.bvh,soma_shapes/soma_base_fit_mhr_params.npz,soma_proportional/bvh/220713/jump_forward_001__A022.bvh,soma_shapes/soma_proportion_fit_mhr_params/A022.npz,g1/csv/220713/jump_forward_001__A022.csv,jump_ff_180_R_001,A022,jump_forward_001,220713,_1,jump_ff_180_R,jumping forward from a standing position,make a small forward leap,person makes a small jump as they move forward,"A person is jumping forward with their right foot, taking the leap.","start right foot, move forward, small jump",jump forward,jump ff 180,neutral,neutral,jumping,standing,1,1,0,0,0,A022,T,180,25,152,32,93,135,174,105,28,51,7,62,30,F,general
|
dataset/seed/metadata/seed_metadata_v004_100_move_name.jsonl
ADDED
|
@@ -0,0 +1,100 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{"move_name": "crouch_idle_001__A147"}
|
| 2 |
+
{"move_name": "high_jump_R_opt_1_001__A280"}
|
| 3 |
+
{"move_name": "crouch_idle_right_R_001__A194"}
|
| 4 |
+
{"move_name": "walk_ff_start_315_R_001__A049"}
|
| 5 |
+
{"move_name": "jog_arc_ccw_stop_001__A029"}
|
| 6 |
+
{"move_name": "walk_ff_stop_360_R_001__A017"}
|
| 7 |
+
{"move_name": "crouch_ff_stop_180_R_001__A195"}
|
| 8 |
+
{"move_name": "turn_jump_000_001__A022"}
|
| 9 |
+
{"move_name": "jog_ff_start_180_R_001__A021"}
|
| 10 |
+
{"move_name": "jog_arc_ccw_start_001__A029"}
|
| 11 |
+
{"move_name": "jog_arc_cw_stop_R_001__A049"}
|
| 12 |
+
{"move_name": "walk_ff_stop_180_R_001__A017"}
|
| 13 |
+
{"move_name": "turn_walk_000_L_001__A021"}
|
| 14 |
+
{"move_name": "jog_ff_start_000_001__A025"}
|
| 15 |
+
{"move_name": "jump_ff_000_001__A021"}
|
| 16 |
+
{"move_name": "jog_ff_start_315_R_001__A050"}
|
| 17 |
+
{"move_name": "turn_jump_090_001__A023"}
|
| 18 |
+
{"move_name": "crouch_turn_360_R_001__A126"}
|
| 19 |
+
{"move_name": "turn_walk_045_L_001__A023"}
|
| 20 |
+
{"move_name": "jump_twice_R_001__A533"}
|
| 21 |
+
{"move_name": "relaxed_fast_walk_ff_180_001__A005"}
|
| 22 |
+
{"move_name": "crouch_ff_stop_360_R_001__A126"}
|
| 23 |
+
{"move_name": "high_jump_full_turn_R_opt_1_001__A479"}
|
| 24 |
+
{"move_name": "high_jump_R_opt_2_001__A476"}
|
| 25 |
+
{"move_name": "jump_around_001__A492"}
|
| 26 |
+
{"move_name": "high_jump_ff_180_R_opt_2_001__A476"}
|
| 27 |
+
{"move_name": "high_jump_ff_180_R_opt_1_001__A476"}
|
| 28 |
+
{"move_name": "walk_ff_start_360_R_fast_001__A447"}
|
| 29 |
+
{"move_name": "jog_arc_cw_start_R_001__A093"}
|
| 30 |
+
{"move_name": "jog_ff_start_360_R_normal_pace_001__A451"}
|
| 31 |
+
{"move_name": "jog_arc_cw_start_R_normal_pace_001__A451"}
|
| 32 |
+
{"move_name": "walk_ff_start_360_R_normal_pace_001__A445"}
|
| 33 |
+
{"move_name": "walk_arc_cw_start_R_normal_pace_001__A443"}
|
| 34 |
+
{"move_name": "walk_ff_start_360_R_very_fast_001__A447"}
|
| 35 |
+
{"move_name": "walk_arc_cw_start_R_very_fast_002__A447"}
|
| 36 |
+
{"move_name": "jog_arc_cw_stop_R_fast_001__A452"}
|
| 37 |
+
{"move_name": "walk_arc_cw_stop_R_fast_002__A447"}
|
| 38 |
+
{"move_name": "jog_arc_cw_stop_R_very_fast_001__A455"}
|
| 39 |
+
{"move_name": "walk_ff_stop_360_R_very_fast_002__A447"}
|
| 40 |
+
{"move_name": "touch_toes_standing_R_001__A533"}
|
| 41 |
+
{"move_name": "jump_ff_360_R_002__A045"}
|
| 42 |
+
{"move_name": "crouch_idle_002__A148"}
|
| 43 |
+
{"move_name": "high_jump_R_opt_1_002__A280"}
|
| 44 |
+
{"move_name": "crouch_idle_right_R_002__A194"}
|
| 45 |
+
{"move_name": "walk_ff_start_315_R_002__A049"}
|
| 46 |
+
{"move_name": "jog_arc_ccw_stop_002__A029"}
|
| 47 |
+
{"move_name": "walk_ff_stop_360_R_002__A049"}
|
| 48 |
+
{"move_name": "crouch_ff_stop_180_R_002__A147"}
|
| 49 |
+
{"move_name": "turn_jump_000_002__A022"}
|
| 50 |
+
{"move_name": "jog_ff_start_180_R_002__A021"}
|
| 51 |
+
{"move_name": "jog_arc_ccw_start_002__A029"}
|
| 52 |
+
{"move_name": "jog_arc_cw_stop_R_002__A045"}
|
| 53 |
+
{"move_name": "walk_ff_stop_180_R_002__A023"}
|
| 54 |
+
{"move_name": "turn_walk_000_L_002__A021"}
|
| 55 |
+
{"move_name": "jog_ff_start_000_002__A025"}
|
| 56 |
+
{"move_name": "jump_ff_000_002__A021"}
|
| 57 |
+
{"move_name": "jog_ff_start_315_R_002__A050"}
|
| 58 |
+
{"move_name": "turn_jump_090_002__A025"}
|
| 59 |
+
{"move_name": "crouch_turn_360_R_002__A125"}
|
| 60 |
+
{"move_name": "turn_walk_045_L_002__A023"}
|
| 61 |
+
{"move_name": "jump_twice_R_002__A533"}
|
| 62 |
+
{"move_name": "relaxed_fast_walk_ff_180_002__A005"}
|
| 63 |
+
{"move_name": "crouch_ff_stop_360_R_002__A126"}
|
| 64 |
+
{"move_name": "high_jump_full_turn_R_opt_1_002__A477"}
|
| 65 |
+
{"move_name": "jump_ff_360_R_003__A045"}
|
| 66 |
+
{"move_name": "crouch_idle_003__A148"}
|
| 67 |
+
{"move_name": "high_jump_R_opt_1_003__A280"}
|
| 68 |
+
{"move_name": "crouch_idle_right_R_003__A194"}
|
| 69 |
+
{"move_name": "walk_ff_start_315_R_003__A049"}
|
| 70 |
+
{"move_name": "jog_arc_ccw_stop_003__A029"}
|
| 71 |
+
{"move_name": "walk_ff_stop_360_R_003__A049"}
|
| 72 |
+
{"move_name": "crouch_ff_stop_180_R_003__A146"}
|
| 73 |
+
{"move_name": "turn_jump_000_003__A022"}
|
| 74 |
+
{"move_name": "jog_ff_start_180_R_003__A022"}
|
| 75 |
+
{"move_name": "jog_arc_ccw_start_003__A029"}
|
| 76 |
+
{"move_name": "jog_arc_cw_stop_R_003__A047"}
|
| 77 |
+
{"move_name": "walk_ff_stop_180_R_003__A025"}
|
| 78 |
+
{"move_name": "turn_walk_000_L_003__A021"}
|
| 79 |
+
{"move_name": "jog_ff_start_000_003__A025"}
|
| 80 |
+
{"move_name": "jump_ff_000_003__A021"}
|
| 81 |
+
{"move_name": "jog_ff_start_315_R_003__A050"}
|
| 82 |
+
{"move_name": "turn_jump_090_003__A023"}
|
| 83 |
+
{"move_name": "crouch_turn_360_R_003__A125"}
|
| 84 |
+
{"move_name": "turn_walk_045_L_003__A023"}
|
| 85 |
+
{"move_name": "jump_twice_R_003__A533"}
|
| 86 |
+
{"move_name": "relaxed_fast_walk_ff_180_003__A005"}
|
| 87 |
+
{"move_name": "crouch_ff_stop_360_R_003__A126"}
|
| 88 |
+
{"move_name": "jump_ff_360_R_004__A045"}
|
| 89 |
+
{"move_name": "crouch_idle_004__A244"}
|
| 90 |
+
{"move_name": "high_jump_R_opt_1_004__A305"}
|
| 91 |
+
{"move_name": "crouch_idle_right_R_101__A125"}
|
| 92 |
+
{"move_name": "walk_ff_start_315_R_101__A174"}
|
| 93 |
+
{"move_name": "kneeling_start_R_001__A021"}
|
| 94 |
+
{"move_name": "kneeling_stop_R_001__A021"}
|
| 95 |
+
{"move_name": "walk_ff_start_180_R_001__A021"}
|
| 96 |
+
{"move_name": "walk_ff_loop_270_R_001__A021"}
|
| 97 |
+
{"move_name": "jog_ff_loop_180_R_001__A017"}
|
| 98 |
+
{"move_name": "walk_ff_loop_180_R_001__A017"}
|
| 99 |
+
{"move_name": "jump_ff_180_R_001__A021"}
|
| 100 |
+
{"move_name": "jump_ff_180_R_001__A022"}
|
flake.lock
ADDED
|
@@ -0,0 +1,27 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"nodes": {
|
| 3 |
+
"nixpkgs": {
|
| 4 |
+
"locked": {
|
| 5 |
+
"lastModified": 1772963539,
|
| 6 |
+
"narHash": "sha256-9jVDGZnvCckTGdYT53d/EfznygLskyLQXYwJLKMPsZs=",
|
| 7 |
+
"owner": "NixOS",
|
| 8 |
+
"repo": "nixpkgs",
|
| 9 |
+
"rev": "9dcb002ca1690658be4a04645215baea8b95f31d",
|
| 10 |
+
"type": "github"
|
| 11 |
+
},
|
| 12 |
+
"original": {
|
| 13 |
+
"owner": "NixOS",
|
| 14 |
+
"ref": "nixos-unstable",
|
| 15 |
+
"repo": "nixpkgs",
|
| 16 |
+
"type": "github"
|
| 17 |
+
}
|
| 18 |
+
},
|
| 19 |
+
"root": {
|
| 20 |
+
"inputs": {
|
| 21 |
+
"nixpkgs": "nixpkgs"
|
| 22 |
+
}
|
| 23 |
+
}
|
| 24 |
+
},
|
| 25 |
+
"root": "root",
|
| 26 |
+
"version": 7
|
| 27 |
+
}
|
flake.nix
ADDED
|
@@ -0,0 +1,105 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
description = "PSI development shells";
|
| 3 |
+
|
| 4 |
+
inputs = {
|
| 5 |
+
self.submodules = true;
|
| 6 |
+
nixpkgs.url = "github:NixOS/nixpkgs/nixos-unstable";
|
| 7 |
+
};
|
| 8 |
+
|
| 9 |
+
outputs = {
|
| 10 |
+
self,
|
| 11 |
+
nixpkgs,
|
| 12 |
+
}:
|
| 13 |
+
let
|
| 14 |
+
system = "x86_64-linux";
|
| 15 |
+
pkgs = import nixpkgs {
|
| 16 |
+
inherit system;
|
| 17 |
+
config = {
|
| 18 |
+
allowUnfree = true;
|
| 19 |
+
cudaSupport = true;
|
| 20 |
+
};
|
| 21 |
+
};
|
| 22 |
+
lib = pkgs.lib;
|
| 23 |
+
runtimeLib = import ./nix/lib/runtime.nix {
|
| 24 |
+
inherit lib pkgs;
|
| 25 |
+
};
|
| 26 |
+
psiBaseRuntime = import ./nix/runtime-base.nix { inherit pkgs; };
|
| 27 |
+
hostGpuRuntime = import ./nix/runtime-host-gpu.nix { inherit pkgs; };
|
| 28 |
+
|
| 29 |
+
commonPackages = [
|
| 30 |
+
pkgs.bashInteractive
|
| 31 |
+
pkgs.coreutils
|
| 32 |
+
pkgs.curl
|
| 33 |
+
pkgs.git
|
| 34 |
+
pkgs.git-lfs
|
| 35 |
+
pkgs.uv
|
| 36 |
+
pkgs.wget
|
| 37 |
+
];
|
| 38 |
+
|
| 39 |
+
commonShellHook = ''
|
| 40 |
+
if [ -f "$HOME/.env" ]; then
|
| 41 |
+
set -a
|
| 42 |
+
. "$HOME/.env"
|
| 43 |
+
set +a
|
| 44 |
+
fi
|
| 45 |
+
|
| 46 |
+
_psi_detect_torch_cuda_arch_list() {
|
| 47 |
+
if [ -n "''${TORCH_CUDA_ARCH_LIST:-}" ]; then
|
| 48 |
+
return 0
|
| 49 |
+
fi
|
| 50 |
+
|
| 51 |
+
if ! command -v nvidia-smi >/dev/null 2>&1; then
|
| 52 |
+
return 1
|
| 53 |
+
fi
|
| 54 |
+
|
| 55 |
+
_psi_compute_caps="$(
|
| 56 |
+
nvidia-smi --query-gpu=compute_cap --format=csv,noheader,nounits 2>/dev/null \
|
| 57 |
+
| sed 's/^[[:space:]]*//; s/[[:space:]]*$//' \
|
| 58 |
+
| sed '/^$/d'
|
| 59 |
+
)"
|
| 60 |
+
|
| 61 |
+
if [ -z "$_psi_compute_caps" ]; then
|
| 62 |
+
return 1
|
| 63 |
+
fi
|
| 64 |
+
|
| 65 |
+
_psi_compute_cap_count="$(printf '%s\n' "$_psi_compute_caps" | sort -u | wc -l)"
|
| 66 |
+
if [ "$_psi_compute_cap_count" -ne 1 ]; then
|
| 67 |
+
return 1
|
| 68 |
+
fi
|
| 69 |
+
|
| 70 |
+
_psi_compute_cap="$(printf '%s\n' "$_psi_compute_caps" | head -n 1)"
|
| 71 |
+
if [ -n "$_psi_compute_cap" ]; then
|
| 72 |
+
export TORCH_CUDA_ARCH_LIST="$_psi_compute_cap+PTX"
|
| 73 |
+
return 0
|
| 74 |
+
fi
|
| 75 |
+
|
| 76 |
+
return 1
|
| 77 |
+
}
|
| 78 |
+
|
| 79 |
+
_psi_detect_torch_cuda_arch_list || true
|
| 80 |
+
unset -f _psi_detect_torch_cuda_arch_list
|
| 81 |
+
'';
|
| 82 |
+
in {
|
| 83 |
+
devShells.${system} = {
|
| 84 |
+
default = runtimeLib.mkRuntimeShell {
|
| 85 |
+
name = "psi";
|
| 86 |
+
runtimes = [ psiBaseRuntime hostGpuRuntime ];
|
| 87 |
+
extraPackages = commonPackages;
|
| 88 |
+
extraShellHook = ''
|
| 89 |
+
${commonShellHook}
|
| 90 |
+
echo "SIMPLE runtime composed into root shell"
|
| 91 |
+
'';
|
| 92 |
+
stdenv = pkgs.gcc13Stdenv;
|
| 93 |
+
};
|
| 94 |
+
|
| 95 |
+
psi = runtimeLib.mkRuntimeShell {
|
| 96 |
+
name = "psi";
|
| 97 |
+
runtimes = [ psiBaseRuntime hostGpuRuntime ];
|
| 98 |
+
extraPackages = commonPackages;
|
| 99 |
+
extraShellHook = commonShellHook;
|
| 100 |
+
stdenv = pkgs.gcc13Stdenv;
|
| 101 |
+
};
|
| 102 |
+
|
| 103 |
+
};
|
| 104 |
+
};
|
| 105 |
+
}
|
infer.sh
ADDED
|
@@ -0,0 +1,39 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# <modify_textop_repr> infer脚本固定到v4仓库。
|
| 2 |
+
# 原逻辑:
|
| 3 |
+
# cd /cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/Psi0-yth
|
| 4 |
+
cd /cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/Psi0-yth-v18
|
| 5 |
+
export PYTHONPATH=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/Psi0-yth-v18/src
|
| 6 |
+
export CUDA_VISIBLE_DEVICES=0
|
| 7 |
+
pkill -f monitor.py
|
| 8 |
+
|
| 9 |
+
# prompt
|
| 10 |
+
# PYTHONPATH=src /cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/envs/Psi0/bin/python scripts/infer/seed_text_to_motion.py \
|
| 11 |
+
# --run-dir /cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/Psi0-yth/runs/posttrain_v1/posttrain.sg.flow1000.consta.lr1.0e-04.b512.gpus8.2604160750 \
|
| 12 |
+
# --ckpt-step 1300 \
|
| 13 |
+
# --output-dir /cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/Psi0-yth/runs/posttrain_v1/2604160750_v2 \
|
| 14 |
+
# --prompt "walk backward naturally and stop smoothly" \
|
| 15 |
+
# --prompt-motion-name walk_backward_stop \
|
| 16 |
+
# --num-output-frames 400 \
|
| 17 |
+
# --num-inference-steps 10 \
|
| 18 |
+
# --device cuda:0
|
| 19 |
+
|
| 20 |
+
# metadata
|
| 21 |
+
# RUN_DIR="/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/envs/.psi-home/runs/posttrain_v1/posttrain.sg.flow1000.consta.lr1.0e-04.b512.gpus8.2604161308"
|
| 22 |
+
# OUTPUT_DIR="/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/Psi0-yth/runs/posttrain_v1/2604161306"
|
| 23 |
+
# RUN_DIR="/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/envs/.psi-home/runs/posttrain_v1/posttrain.sg.flow1000.consta.lr1.0e-04.b512.gpus8.2604161308"
|
| 24 |
+
# OUTPUT_DIR="/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/Psi0-yth/runs/posttrain_v1/2604161308"
|
| 25 |
+
RUN_DIR="/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/envs/.psi-home/runs/posttrain_v1/posttrain.sg.flow1000.consta.lr1.0e-04.b512.gpus8.2604170218"
|
| 26 |
+
# <modify_states> 默认infer输出改到v2仓库。
|
| 27 |
+
# 原逻辑:
|
| 28 |
+
# OUTPUT_DIR="/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/Psi0-yth/runs/posttrain_v1/2604170218"
|
| 29 |
+
OUTPUT_DIR="/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/Psi0-yth-v18/runs/posttrain_v1/2604170218"
|
| 30 |
+
# RUN_DIR="/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/envs/.psi-home/runs/posttrain_v1/posttrain.sg.flow1000.consta.lr1.0e-04.b512.gpus8.2604170323"
|
| 31 |
+
# OUTPUT_DIR="/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/Psi0-yth/runs/posttrain_v1/2604170323"
|
| 32 |
+
/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/envs/Psi0/bin/python scripts/infer/seed_text_to_motion.py \
|
| 33 |
+
--run-dir $RUN_DIR \
|
| 34 |
+
--ckpt-step 55000 \
|
| 35 |
+
--output-dir $OUTPUT_DIR \
|
| 36 |
+
--num-samples 1 \
|
| 37 |
+
--start-index 8 \
|
| 38 |
+
--num-inference-steps 10 \
|
| 39 |
+
--device cuda:0
|
infer_prompt.sh
ADDED
|
@@ -0,0 +1,213 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
cd /cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/Psi0-yth-v18
|
| 2 |
+
source /cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/Psi0-yth-v15/.venv-psi/bin/activate
|
| 3 |
+
[ -f .env ] && source .env
|
| 4 |
+
|
| 5 |
+
export PYTHONPATH=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/Psi0-yth-v18/src
|
| 6 |
+
export CUDA_VISIBLE_DEVICES=0
|
| 7 |
+
export PSI_INFER_FLOAT32=1
|
| 8 |
+
pkill -f monitor.py
|
| 9 |
+
|
| 10 |
+
# 改这里选择推理类型:audio | audio_command | audio_command_only | text | video | video_clip
|
| 11 |
+
MODE=video_clip
|
| 12 |
+
|
| 13 |
+
NUM_INFERENCE_STEPS=10
|
| 14 |
+
DEVICE=cuda:0
|
| 15 |
+
|
| 16 |
+
RUN_TIMESTAMP=2606041200
|
| 17 |
+
# RUN_TIMESTAMP=2606041447
|
| 18 |
+
RUN_PARENT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/Psi0-yth-v18/runs/posttrain_v1_trimodal
|
| 19 |
+
RUN_DIR=$RUN_PARENT/posttrain.sg.flow1000.cwml.lr1.0e-04.b256.gpus8.$RUN_TIMESTAMP
|
| 20 |
+
CKPT_STEP=best
|
| 21 |
+
|
| 22 |
+
# 普通 audio / video 默认样例
|
| 23 |
+
MOTION_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/GR00T-WholeBodyControl/gear_sonic_deploy/reference/converted/psi0_dance/gvhmr
|
| 24 |
+
VIDEO_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/dance_1723_video_psi0/dance_1723_video_10s
|
| 25 |
+
MOTION_NAME=movies__F-SNBR4jhY_0007
|
| 26 |
+
# MOTION_DIR=$MOTION_ROOT/$MOTION_NAME
|
| 27 |
+
# VIDEO_PATH=$VIDEO_ROOT/$MOTION_NAME.mp4
|
| 28 |
+
# VIDEO_PATH=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/videogen/motion_example_0506_demo/original_videos/00000/0h7AlrKLslw_00056_0_313.mp4
|
| 29 |
+
# VIDEO_PATH=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/dance_1723_video_psi0/dance_1723_video_10s/movies__F-SNBR4jhY_0007.mp4
|
| 30 |
+
|
| 31 |
+
MOTION_DIR=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/GR00T-WholeBodyControl/gear_sonic_deploy/reference/converted/psi0_dance/gvhmr/movies__6RU6bQLqVk_0000
|
| 32 |
+
VIDEO_PATH=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/Psi0-yth-v18/kun3.mp4
|
| 33 |
+
|
| 34 |
+
# MOTION_DIR=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/videogen/psi0_video_clip_50f_rand8/dance_1723_video_10s/motion/movies__F-SNBR4jhY_0007.chunk00.f000160
|
| 35 |
+
# VIDEO_PATH=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/videogen/psi0_video_clip_50f_rand8/dance_1723_video_10s/videos/movies__F-SNBR4jhY_0007.chunk00.f000160.mp4
|
| 36 |
+
|
| 37 |
+
# MOTION_DIR=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/videogen/psi0_video_clip_50f_rand32_full/motion_example_0506_demo/motion/converted_motion__00000___dFAYYMJbWk_00193_268_382.chunk00.f000000
|
| 38 |
+
# VIDEO_PATH=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/videogen/psi0_video_clip_50f_rand32_full/motion_example_0506_demo/videos/converted_motion__00000___dFAYYMJbWk_00193_268_382.chunk00.f000000.mp4
|
| 39 |
+
|
| 40 |
+
# audio_command 默认样例;AUDIO_COMMAND_AUDIO_PATH 是目录时会随机抽一条 wav
|
| 41 |
+
# AUDIO_COMMAND_MOTION_DIR=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/Psi0/dataset/seed_all/g1/csv_sonic/230530/sitting_legs_bend_arms_front_loop_R_001__A410
|
| 42 |
+
# AUDIO_COMMAND_AUDIO_PATH=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/qwen3-tts/seed_audio/003856
|
| 43 |
+
# AUDIO_COMMAND_AUDIO_PATH=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/qwen3-tts/seed_audio/000017
|
| 44 |
+
AUDIO_COMMAND_MOTION_DIR=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/GR00T-WholeBodyControl/gear_sonic_deploy/reference/converted/psi0_dance/gvhmr/movies_1mzl0Zh614A_0000
|
| 45 |
+
AUDIO_COMMAND_PROMPT="Generate a motion that follows the spoken instruction in the audio"
|
| 46 |
+
AUDIO_COMMAND_AUDIO_PATH=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/Psi0-yth-v18/jump.wav
|
| 47 |
+
|
| 48 |
+
# text 默认样例
|
| 49 |
+
TEXT_PROMPT="Generate the following motion sequence: A person jog forward."
|
| 50 |
+
TEXT_START_INDEX=0
|
| 51 |
+
TEXT_NUM_OUTPUT_FRAMES=300
|
| 52 |
+
PROMPT_MOTION_NAME=text_prompt
|
| 53 |
+
|
| 54 |
+
case "$MODE" in
|
| 55 |
+
audio|audio_text|audio+text|audio_only|audio_pure)
|
| 56 |
+
MODE_NAME=audio
|
| 57 |
+
AUDIO_CONFIG_KEY=audio
|
| 58 |
+
OUTPUT_DIR=$RUN_PARENT/$RUN_TIMESTAMP.$MODE_NAME.test
|
| 59 |
+
PROMPT="Generate a dance sequence that matches the provided music."
|
| 60 |
+
if [ "$MODE" = "audio_only" ] || [ "$MODE" = "audio_pure" ]; then
|
| 61 |
+
PROMPT=""
|
| 62 |
+
fi
|
| 63 |
+
|
| 64 |
+
# rm -rf "$OUTPUT_DIR"
|
| 65 |
+
|
| 66 |
+
echo "Mode: $MODE_NAME"
|
| 67 |
+
echo "Audio config key: $AUDIO_CONFIG_KEY"
|
| 68 |
+
echo "Run dir: $RUN_DIR"
|
| 69 |
+
echo "Ckpt step: $CKPT_STEP"
|
| 70 |
+
echo "Prompt: $PROMPT"
|
| 71 |
+
echo "Motion dir: $MOTION_DIR"
|
| 72 |
+
echo "Video path: $VIDEO_PATH"
|
| 73 |
+
echo "Output dir: $OUTPUT_DIR"
|
| 74 |
+
|
| 75 |
+
python scripts/infer/seed_audio_to_motion.py \
|
| 76 |
+
--run-dir "$RUN_DIR" \
|
| 77 |
+
--ckpt-step "$CKPT_STEP" \
|
| 78 |
+
--output-dir "$OUTPUT_DIR" \
|
| 79 |
+
--motion-dir "$MOTION_DIR" \
|
| 80 |
+
--video-path "$VIDEO_PATH" \
|
| 81 |
+
--audio-config-key "$AUDIO_CONFIG_KEY" \
|
| 82 |
+
--prompt "$PROMPT" \
|
| 83 |
+
--num-inference-steps "$NUM_INFERENCE_STEPS" \
|
| 84 |
+
--device "$DEVICE" || return 1 2>/dev/null || exit 1
|
| 85 |
+
;;
|
| 86 |
+
audio_command|audio_command_text|audio_command_only|command_audio)
|
| 87 |
+
MODE_NAME=audio_command
|
| 88 |
+
AUDIO_CONFIG_KEY=audio_command
|
| 89 |
+
OUTPUT_DIR=$RUN_PARENT/$RUN_TIMESTAMP.$MODE_NAME.test
|
| 90 |
+
MOTION_DIR=$AUDIO_COMMAND_MOTION_DIR
|
| 91 |
+
VIDEO_PATH=$AUDIO_COMMAND_AUDIO_PATH
|
| 92 |
+
PROMPT=$AUDIO_COMMAND_PROMPT
|
| 93 |
+
if [ "$MODE" = "audio_command_only" ]; then
|
| 94 |
+
PROMPT=""
|
| 95 |
+
fi
|
| 96 |
+
|
| 97 |
+
# rm -rf "$OUTPUT_DIR"
|
| 98 |
+
|
| 99 |
+
echo "Mode: $MODE_NAME"
|
| 100 |
+
echo "Audio config key: $AUDIO_CONFIG_KEY"
|
| 101 |
+
echo "Run dir: $RUN_DIR"
|
| 102 |
+
echo "Ckpt step: $CKPT_STEP"
|
| 103 |
+
echo "Prompt: $PROMPT"
|
| 104 |
+
echo "Motion dir: $MOTION_DIR"
|
| 105 |
+
echo "Video path: $VIDEO_PATH"
|
| 106 |
+
echo "Output dir: $OUTPUT_DIR"
|
| 107 |
+
|
| 108 |
+
python scripts/infer/seed_audio_to_motion.py \
|
| 109 |
+
--run-dir "$RUN_DIR" \
|
| 110 |
+
--ckpt-step "$CKPT_STEP" \
|
| 111 |
+
--output-dir "$OUTPUT_DIR" \
|
| 112 |
+
--motion-dir "$MOTION_DIR" \
|
| 113 |
+
--video-path "$VIDEO_PATH" \
|
| 114 |
+
--audio-config-key "$AUDIO_CONFIG_KEY" \
|
| 115 |
+
--prompt "$PROMPT" \
|
| 116 |
+
--num-inference-steps "$NUM_INFERENCE_STEPS" \
|
| 117 |
+
--device "$DEVICE" || return 1 2>/dev/null || exit 1
|
| 118 |
+
;;
|
| 119 |
+
text|pure_text)
|
| 120 |
+
MODE_NAME=text
|
| 121 |
+
OUTPUT_DIR=$RUN_PARENT/$RUN_TIMESTAMP.$MODE_NAME.test
|
| 122 |
+
|
| 123 |
+
# rm -rf "$OUTPUT_DIR"
|
| 124 |
+
|
| 125 |
+
echo "Mode: $MODE_NAME"
|
| 126 |
+
echo "Run dir: $RUN_DIR"
|
| 127 |
+
echo "Ckpt step: $CKPT_STEP"
|
| 128 |
+
echo "Prompt: $TEXT_PROMPT"
|
| 129 |
+
echo "Prompt motion name: $PROMPT_MOTION_NAME"
|
| 130 |
+
echo "Start index: $TEXT_START_INDEX"
|
| 131 |
+
echo "Num output frames: $TEXT_NUM_OUTPUT_FRAMES"
|
| 132 |
+
echo "Output dir: $OUTPUT_DIR"
|
| 133 |
+
|
| 134 |
+
python scripts/infer/seed_text_to_motion.py \
|
| 135 |
+
--run-dir "$RUN_DIR" \
|
| 136 |
+
--ckpt-step "$CKPT_STEP" \
|
| 137 |
+
--output-dir "$OUTPUT_DIR" \
|
| 138 |
+
--prompt "$TEXT_PROMPT" \
|
| 139 |
+
--prompt-motion-name "$PROMPT_MOTION_NAME" \
|
| 140 |
+
--init-motion-dir "$MOTION_DIR" \
|
| 141 |
+
--start-index "$TEXT_START_INDEX" \
|
| 142 |
+
--num-output-frames "$TEXT_NUM_OUTPUT_FRAMES" \
|
| 143 |
+
--num-inference-steps "$NUM_INFERENCE_STEPS" \
|
| 144 |
+
--force-text-only \
|
| 145 |
+
--device "$DEVICE" || return 1 2>/dev/null || exit 1
|
| 146 |
+
;;
|
| 147 |
+
video|video_text|video+text|video_only|video_pure|video_clip|video_clip_text|video_clip_only)
|
| 148 |
+
if [ "$MODE" = "video_clip" ] || [ "$MODE" = "video_clip_text" ] || [ "$MODE" = "video_clip_only" ]; then
|
| 149 |
+
MODE_NAME=video_clip
|
| 150 |
+
VIDEO_CONFIG_KEY=video_clip
|
| 151 |
+
VIDEO_CLIP_ARGS=(--video-config-key video_clip --video-clip)
|
| 152 |
+
else
|
| 153 |
+
MODE_NAME=video
|
| 154 |
+
VIDEO_CONFIG_KEY=video
|
| 155 |
+
VIDEO_CLIP_ARGS=(--video-config-key video)
|
| 156 |
+
fi
|
| 157 |
+
OUTPUT_DIR=$RUN_PARENT/$RUN_TIMESTAMP.$MODE_NAME.test
|
| 158 |
+
PROMPT="Generate a motion sequence that matches the provided video."
|
| 159 |
+
if [ "$MODE" = "video_only" ] || [ "$MODE" = "video_pure" ] || [ "$MODE" = "video_clip_only" ]; then
|
| 160 |
+
PROMPT=""
|
| 161 |
+
fi
|
| 162 |
+
|
| 163 |
+
# rm -rf "$OUTPUT_DIR"
|
| 164 |
+
|
| 165 |
+
echo "Mode: $MODE_NAME"
|
| 166 |
+
echo "Video config key: $VIDEO_CONFIG_KEY"
|
| 167 |
+
echo "Run dir: $RUN_DIR"
|
| 168 |
+
echo "Ckpt step: $CKPT_STEP"
|
| 169 |
+
echo "Prompt: $PROMPT"
|
| 170 |
+
echo "Motion dir: $MOTION_DIR"
|
| 171 |
+
echo "Video path: $VIDEO_PATH"
|
| 172 |
+
echo "Output dir: $OUTPUT_DIR"
|
| 173 |
+
|
| 174 |
+
python scripts/infer/seed_video_to_motion.py \
|
| 175 |
+
--run-dir "$RUN_DIR" \
|
| 176 |
+
--ckpt-step "$CKPT_STEP" \
|
| 177 |
+
--output-dir "$OUTPUT_DIR" \
|
| 178 |
+
--motion-dir "$MOTION_DIR" \
|
| 179 |
+
--video-path "$VIDEO_PATH" \
|
| 180 |
+
--prompt "$PROMPT" \
|
| 181 |
+
--num-inference-steps "$NUM_INFERENCE_STEPS" \
|
| 182 |
+
"${VIDEO_CLIP_ARGS[@]}" \
|
| 183 |
+
--device "$DEVICE" || return 1 2>/dev/null || exit 1
|
| 184 |
+
;;
|
| 185 |
+
*)
|
| 186 |
+
echo "ERROR: unknown mode '$MODE'. Use 'audio', 'audio_command', 'audio_command_only', 'text', 'video', or 'video_clip'."
|
| 187 |
+
exit 1
|
| 188 |
+
;;
|
| 189 |
+
esac
|
| 190 |
+
|
| 191 |
+
deactivate
|
| 192 |
+
unset PYTHONPATH
|
| 193 |
+
cd /cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/GR00T-WholeBodyControl/gear_sonic
|
| 194 |
+
export PYOPENGL_PLATFORM="egl"
|
| 195 |
+
export MUJOCO_GL="egl"
|
| 196 |
+
export QT_X11_NO_MITSHM=1
|
| 197 |
+
xvfb-run -a -s "-screen 0 1280x1024x24 -extension MIT-SHM" /cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/envs/gmr/bin/python scripts/batch_render_reference_motion_mujoco.py \
|
| 198 |
+
--motion-dir "$OUTPUT_DIR" \
|
| 199 |
+
--source-fps 50 \
|
| 200 |
+
--target-fps 30 \
|
| 201 |
+
--dof-order isaaclab
|
| 202 |
+
|
| 203 |
+
# deactivate
|
| 204 |
+
# unset PYTHONPATH
|
| 205 |
+
# cd /cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/GR00T-WholeBodyControl/gear_sonic
|
| 206 |
+
# export PYOPENGL_PLATFORM="egl"
|
| 207 |
+
# export MUJOCO_GL="egl"
|
| 208 |
+
# export QT_X11_NO_MITSHM=1
|
| 209 |
+
# xvfb-run -a -s "-screen 0 1280x1024x24 -extension MIT-SHM" /cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/envs/gmr/bin/python scripts/batch_render_reference_motion_mujoco.py \
|
| 210 |
+
# --motion-dir "/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/videogen/psi0_video_clip_50f_rand32_full/motion_example_0506_demo/motion/converted_motion__00000___dFAYYMJbWk_00193_268_382.chunk00.f000000" \
|
| 211 |
+
# --source-fps 50 \
|
| 212 |
+
# --target-fps 30 \
|
| 213 |
+
# --dof-order isaaclab
|
infer_prompt_distill.sh
ADDED
|
@@ -0,0 +1,104 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#!/usr/bin/env bash
|
| 2 |
+
pkill -f monitor.py
|
| 3 |
+
|
| 4 |
+
############################
|
| 5 |
+
# Edit config only here.
|
| 6 |
+
############################
|
| 7 |
+
|
| 8 |
+
STAGE=student
|
| 9 |
+
|
| 10 |
+
# RUN_DIR=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/Psi0-yth-v18/runs/posttrain_v1_motion_distill/posttrain.motion_distill_teacher.sg.flow1000.cwml.lr1.0e-04.b512.gpus8.2605302159
|
| 11 |
+
RUN_DIR=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/Psi0-yth-v18/runs/posttrain_v1_motion_distill/posttrain.motion_distill_student.sg.flow1000.cwml.lr1.0e-04.b256.gpus8.2605310757
|
| 12 |
+
CKPT_STEP=best
|
| 13 |
+
# OUTPUT_DIR=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/Psi0-yth-v18/runs/posttrain_v1_motion_distill/teacher_reconstruct.test
|
| 14 |
+
OUTPUT_DIR=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/Psi0-yth-v18/runs/posttrain_v1_motion_distill/student_infer.test
|
| 15 |
+
|
| 16 |
+
MOTION_DIR=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/videogen/psi0_video_clip_50f_rand8/dance_1723_video_10s/motion/movies__F-SNBR4jhY_0007.chunk00.f000160
|
| 17 |
+
VIDEO_PATH=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/videogen/psi0_video_clip_50f_rand8/dance_1723_video_10s/videos/movies__F-SNBR4jhY_0007.chunk00.f000160.mp4
|
| 18 |
+
PROMPT="Generate a motion sequence that matches the provided video."
|
| 19 |
+
|
| 20 |
+
NUM_OUTPUT_FRAMES=0
|
| 21 |
+
NUM_INFERENCE_STEPS=10
|
| 22 |
+
DEVICE=cuda:0
|
| 23 |
+
CUDA_VISIBLE_DEVICES=0
|
| 24 |
+
PSI_INFER_FLOAT32=1
|
| 25 |
+
|
| 26 |
+
VIDEO_CONFIG_KEY=video_clip
|
| 27 |
+
VIDEO_CLIP=1
|
| 28 |
+
RENDER=1
|
| 29 |
+
|
| 30 |
+
REPO_DIR=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/Psi0-yth-v18
|
| 31 |
+
GEAR_SONIC_DIR=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/GR00T-WholeBodyControl/gear_sonic
|
| 32 |
+
GMR_PYTHON=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/envs/gmr/bin/python
|
| 33 |
+
|
| 34 |
+
############################
|
| 35 |
+
|
| 36 |
+
cd "$REPO_DIR"
|
| 37 |
+
|
| 38 |
+
source .venv-psi/bin/activate
|
| 39 |
+
[ -f .env ] && source .env
|
| 40 |
+
|
| 41 |
+
export PYTHONPATH=$REPO_DIR/src
|
| 42 |
+
export CUDA_VISIBLE_DEVICES
|
| 43 |
+
export PSI_INFER_FLOAT32
|
| 44 |
+
|
| 45 |
+
pkill -f monitor.py 2>/dev/null || true
|
| 46 |
+
rm -rf "$OUTPUT_DIR"
|
| 47 |
+
|
| 48 |
+
echo "Stage: $STAGE"
|
| 49 |
+
echo "Run dir: $RUN_DIR"
|
| 50 |
+
echo "Ckpt step: $CKPT_STEP"
|
| 51 |
+
echo "Motion dir: $MOTION_DIR"
|
| 52 |
+
echo "Video path: $VIDEO_PATH"
|
| 53 |
+
echo "Output dir: $OUTPUT_DIR"
|
| 54 |
+
echo "Num output frames: $NUM_OUTPUT_FRAMES"
|
| 55 |
+
echo "Num inference steps: $NUM_INFERENCE_STEPS"
|
| 56 |
+
echo "Device: $DEVICE"
|
| 57 |
+
|
| 58 |
+
if [ "$STAGE" = "teacher" ]; then
|
| 59 |
+
python scripts/infer/seed_motion_teacher_reconstruct.py \
|
| 60 |
+
--run-dir "$RUN_DIR" \
|
| 61 |
+
--ckpt-step "$CKPT_STEP" \
|
| 62 |
+
--output-dir "$OUTPUT_DIR" \
|
| 63 |
+
--motion-dir "$MOTION_DIR" \
|
| 64 |
+
--num-output-frames "$NUM_OUTPUT_FRAMES" \
|
| 65 |
+
--num-inference-steps "$NUM_INFERENCE_STEPS" \
|
| 66 |
+
--device "$DEVICE"
|
| 67 |
+
elif [ "$STAGE" = "student" ]; then
|
| 68 |
+
VIDEO_ARGS=(--video-config-key "$VIDEO_CONFIG_KEY")
|
| 69 |
+
if [ "$VIDEO_CLIP" = "1" ]; then
|
| 70 |
+
VIDEO_ARGS+=(--video-clip)
|
| 71 |
+
fi
|
| 72 |
+
|
| 73 |
+
python scripts/infer/seed_video_to_motion.py \
|
| 74 |
+
--run-dir "$RUN_DIR" \
|
| 75 |
+
--ckpt-step "$CKPT_STEP" \
|
| 76 |
+
--output-dir "$OUTPUT_DIR" \
|
| 77 |
+
--motion-dir "$MOTION_DIR" \
|
| 78 |
+
--video-path "$VIDEO_PATH" \
|
| 79 |
+
--prompt "$PROMPT" \
|
| 80 |
+
--num-output-frames "$NUM_OUTPUT_FRAMES" \
|
| 81 |
+
--num-inference-steps "$NUM_INFERENCE_STEPS" \
|
| 82 |
+
"${VIDEO_ARGS[@]}" \
|
| 83 |
+
--device "$DEVICE"
|
| 84 |
+
else
|
| 85 |
+
echo "ERROR: STAGE must be teacher or student."
|
| 86 |
+
exit 1
|
| 87 |
+
fi
|
| 88 |
+
|
| 89 |
+
deactivate
|
| 90 |
+
unset PYTHONPATH
|
| 91 |
+
|
| 92 |
+
if [ "$RENDER" != "1" ]; then
|
| 93 |
+
exit 0
|
| 94 |
+
fi
|
| 95 |
+
|
| 96 |
+
cd "$GEAR_SONIC_DIR"
|
| 97 |
+
export PYOPENGL_PLATFORM=egl
|
| 98 |
+
export MUJOCO_GL=egl
|
| 99 |
+
export QT_X11_NO_MITSHM=1
|
| 100 |
+
xvfb-run -a -s "-screen 0 1280x1024x24 -extension MIT-SHM" "$GMR_PYTHON" scripts/batch_render_reference_motion_mujoco.py \
|
| 101 |
+
--motion-dir "$OUTPUT_DIR" \
|
| 102 |
+
--source-fps 50 \
|
| 103 |
+
--target-fps 30 \
|
| 104 |
+
--dof-order isaaclab
|
infer_prompt_short.sh
ADDED
|
@@ -0,0 +1,157 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#!/usr/bin/env bash
|
| 2 |
+
set -euo pipefail
|
| 3 |
+
|
| 4 |
+
cd /cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/Psi0-yth-v18
|
| 5 |
+
source .venv-psi/bin/activate
|
| 6 |
+
[ -f .env ] && source .env
|
| 7 |
+
|
| 8 |
+
export PYTHONPATH=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/Psi0-yth-v18/src
|
| 9 |
+
export CUDA_VISIBLE_DEVICES=0
|
| 10 |
+
export PSI_INFER_FLOAT32=1
|
| 11 |
+
pkill -f monitor.py || true
|
| 12 |
+
|
| 13 |
+
# 改这里:video2motion 只 rollout 多少个实际执行 chunk。
|
| 14 |
+
SHORT_NUM_CHUNKS=2
|
| 15 |
+
# 改这里选择短推理类型:video | video_only | video_clip | video_clip_only
|
| 16 |
+
MODE=video_clip
|
| 17 |
+
|
| 18 |
+
NUM_INFERENCE_STEPS=10
|
| 19 |
+
DEVICE=cuda:0
|
| 20 |
+
|
| 21 |
+
RUN_TIMESTAMP=2605211455
|
| 22 |
+
RUN_PARENT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/Psi0-yth-v18/runs/posttrain_v1_trimodal
|
| 23 |
+
RUN_DIR=$RUN_PARENT/posttrain.sg.flow1000.cwml.lr1.0e-04.b512.gpus8.$RUN_TIMESTAMP
|
| 24 |
+
CKPT_STEP=best
|
| 25 |
+
|
| 26 |
+
MOTION_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/GR00T-WholeBodyControl/gear_sonic_deploy/reference/converted/psi0_dance/gvhmr
|
| 27 |
+
VIDEO_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/dance_1723_video_psi0/dance_1723_video_10s
|
| 28 |
+
MOTION_NAME=movies_1mzl0Zh614A_0000
|
| 29 |
+
MOTION_DIR=$MOTION_ROOT/$MOTION_NAME
|
| 30 |
+
# VIDEO_PATH=$VIDEO_ROOT/$MOTION_NAME.mp4
|
| 31 |
+
# VIDEO_PATH=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/videogen/motion_example_0506_demo/original_videos/00000/0h7AlrKLslw_00056_0_313.mp4
|
| 32 |
+
VIDEO_PATH=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/dance_1723_video_psi0/dance_1723_video_10s/movies__F-SNBR4jhY_0007.mp4
|
| 33 |
+
|
| 34 |
+
PROMPT="Generate a motion sequence that matches the provided video."
|
| 35 |
+
# PROMPT=""
|
| 36 |
+
TARGET_FPS=50
|
| 37 |
+
|
| 38 |
+
if [ "$SHORT_NUM_CHUNKS" -le 0 ]; then
|
| 39 |
+
echo "ERROR: SHORT_NUM_CHUNKS must be positive, got $SHORT_NUM_CHUNKS"
|
| 40 |
+
exit 1
|
| 41 |
+
fi
|
| 42 |
+
|
| 43 |
+
case "$MODE" in
|
| 44 |
+
video|video_text|video+text|video_only|video_pure)
|
| 45 |
+
MODE_NAME=video
|
| 46 |
+
VIDEO_CONFIG_KEY=video
|
| 47 |
+
VIDEO_CLIP_ARGS=(--video-config-key video)
|
| 48 |
+
;;
|
| 49 |
+
video_clip|video_clip_text|video_clip_only)
|
| 50 |
+
MODE_NAME=video_clip
|
| 51 |
+
VIDEO_CONFIG_KEY=video_clip
|
| 52 |
+
VIDEO_CLIP_ARGS=(--video-config-key video_clip --video-clip --video-clip-motion-fps "$TARGET_FPS")
|
| 53 |
+
;;
|
| 54 |
+
*)
|
| 55 |
+
echo "ERROR: unknown mode '$MODE'. Use 'video' or 'video_clip'."
|
| 56 |
+
exit 1
|
| 57 |
+
;;
|
| 58 |
+
esac
|
| 59 |
+
|
| 60 |
+
if [ "$MODE" = "video_only" ] || [ "$MODE" = "video_pure" ] || [ "$MODE" = "video_clip_only" ]; then
|
| 61 |
+
PROMPT=""
|
| 62 |
+
fi
|
| 63 |
+
|
| 64 |
+
RUN_CONFIG=$RUN_DIR/run_config.json
|
| 65 |
+
if [ ! -f "$RUN_CONFIG" ]; then
|
| 66 |
+
echo "ERROR: run_config.json not found: $RUN_CONFIG"
|
| 67 |
+
exit 1
|
| 68 |
+
fi
|
| 69 |
+
|
| 70 |
+
read -r RTC_ENABLED ACTION_HORIZON EXEC_HORIZON <<< "$(python -c 'import json, sys
|
| 71 |
+
cfg = json.load(open(sys.argv[1], encoding="utf-8"))
|
| 72 |
+
model = cfg.get("model", {})
|
| 73 |
+
rtc = bool(model.get("rtc", False))
|
| 74 |
+
action_horizon = int(model.get("action_chunk_size", 30))
|
| 75 |
+
if rtc:
|
| 76 |
+
exec_horizon = int(model.get("action_exec_horizon", action_horizon))
|
| 77 |
+
else:
|
| 78 |
+
exec_horizon = action_horizon
|
| 79 |
+
print(int(rtc), action_horizon, exec_horizon)' "$RUN_CONFIG")"
|
| 80 |
+
NUM_OUTPUT_FRAMES=$((SHORT_NUM_CHUNKS * EXEC_HORIZON))
|
| 81 |
+
if [ "$MODE_NAME" = "video_clip" ]; then
|
| 82 |
+
VIDEO_INPUT_FRAMES=$(((SHORT_NUM_CHUNKS - 1) * EXEC_HORIZON + ACTION_HORIZON))
|
| 83 |
+
else
|
| 84 |
+
VIDEO_INPUT_FRAMES=$NUM_OUTPUT_FRAMES
|
| 85 |
+
fi
|
| 86 |
+
OUTPUT_DIR=$RUN_PARENT/$RUN_TIMESTAMP.$MODE_NAME.short_${SHORT_NUM_CHUNKS}chunks
|
| 87 |
+
VIDEO_DURATION_SEC=$(python -c 'import sys
|
| 88 |
+
frames = int(sys.argv[1])
|
| 89 |
+
fps = float(sys.argv[2])
|
| 90 |
+
print(f"{frames / fps:.6f}")' "$VIDEO_INPUT_FRAMES" "$TARGET_FPS")
|
| 91 |
+
SOURCE_VIDEO_DURATION_SEC=$(ffprobe -v error -show_entries format=duration -of default=nk=1:nw=1 "$VIDEO_PATH")
|
| 92 |
+
python -c 'import sys
|
| 93 |
+
need = float(sys.argv[1])
|
| 94 |
+
have = float(sys.argv[2])
|
| 95 |
+
if have + 1e-3 < need:
|
| 96 |
+
raise SystemExit(f"ERROR: source video is shorter than requested clip: need {need:.6f}s, have {have:.6f}s")' \
|
| 97 |
+
"$VIDEO_DURATION_SEC" "$SOURCE_VIDEO_DURATION_SEC"
|
| 98 |
+
SHORT_INPUT_DIR=$RUN_PARENT/$RUN_TIMESTAMP.$MODE_NAME.short_inputs
|
| 99 |
+
VIDEO_BASENAME=$(basename "$VIDEO_PATH")
|
| 100 |
+
SHORT_VIDEO_PATH=$SHORT_INPUT_DIR/${VIDEO_BASENAME%.*}.short_${SHORT_NUM_CHUNKS}chunks.mp4
|
| 101 |
+
|
| 102 |
+
rm -rf "$OUTPUT_DIR"
|
| 103 |
+
mkdir -p "$SHORT_INPUT_DIR"
|
| 104 |
+
ffmpeg -hide_banner -loglevel error -y \
|
| 105 |
+
-ss 0 \
|
| 106 |
+
-t "$VIDEO_DURATION_SEC" \
|
| 107 |
+
-i "$VIDEO_PATH" \
|
| 108 |
+
-map 0:v:0 \
|
| 109 |
+
-an \
|
| 110 |
+
-c:v libx264 \
|
| 111 |
+
-pix_fmt yuv420p \
|
| 112 |
+
"$SHORT_VIDEO_PATH"
|
| 113 |
+
CROPPED_VIDEO_DURATION_SEC=$(ffprobe -v error -show_entries format=duration -of default=nk=1:nw=1 "$SHORT_VIDEO_PATH")
|
| 114 |
+
|
| 115 |
+
echo "Mode: $MODE_NAME short"
|
| 116 |
+
echo "Video config key: $VIDEO_CONFIG_KEY"
|
| 117 |
+
echo "Short chunks: $SHORT_NUM_CHUNKS"
|
| 118 |
+
echo "RTC enabled: $RTC_ENABLED"
|
| 119 |
+
echo "Action horizon: $ACTION_HORIZON"
|
| 120 |
+
echo "RTC overlap frames: $((ACTION_HORIZON - EXEC_HORIZON))"
|
| 121 |
+
echo "Exec horizon: $EXEC_HORIZON"
|
| 122 |
+
echo "Num output frames: $NUM_OUTPUT_FRAMES"
|
| 123 |
+
echo "Video input frames: $VIDEO_INPUT_FRAMES"
|
| 124 |
+
echo "Target fps: $TARGET_FPS"
|
| 125 |
+
echo "Short video seconds: $VIDEO_DURATION_SEC"
|
| 126 |
+
echo "Cropped video seconds: $CROPPED_VIDEO_DURATION_SEC"
|
| 127 |
+
echo "Run dir: $RUN_DIR"
|
| 128 |
+
echo "Ckpt step: $CKPT_STEP"
|
| 129 |
+
echo "Prompt: $PROMPT"
|
| 130 |
+
echo "Motion dir: $MOTION_DIR"
|
| 131 |
+
echo "Video path: $VIDEO_PATH"
|
| 132 |
+
echo "Short video path: $SHORT_VIDEO_PATH"
|
| 133 |
+
echo "Output dir: $OUTPUT_DIR"
|
| 134 |
+
|
| 135 |
+
python scripts/infer/seed_video_to_motion.py \
|
| 136 |
+
--run-dir "$RUN_DIR" \
|
| 137 |
+
--ckpt-step "$CKPT_STEP" \
|
| 138 |
+
--output-dir "$OUTPUT_DIR" \
|
| 139 |
+
--motion-dir "$MOTION_DIR" \
|
| 140 |
+
--video-path "$SHORT_VIDEO_PATH" \
|
| 141 |
+
--prompt "$PROMPT" \
|
| 142 |
+
--num-output-frames "$NUM_OUTPUT_FRAMES" \
|
| 143 |
+
--num-inference-steps "$NUM_INFERENCE_STEPS" \
|
| 144 |
+
"${VIDEO_CLIP_ARGS[@]}" \
|
| 145 |
+
--device "$DEVICE"
|
| 146 |
+
|
| 147 |
+
deactivate
|
| 148 |
+
unset PYTHONPATH
|
| 149 |
+
cd /cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/GR00T-WholeBodyControl/gear_sonic
|
| 150 |
+
export PYOPENGL_PLATFORM="egl"
|
| 151 |
+
export MUJOCO_GL="egl"
|
| 152 |
+
export QT_X11_NO_MITSHM=1
|
| 153 |
+
xvfb-run -a -s "-screen 0 1280x1024x24 -extension MIT-SHM" /cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/envs/gmr/bin/python scripts/batch_render_reference_motion_mujoco.py \
|
| 154 |
+
--motion-dir "$OUTPUT_DIR" \
|
| 155 |
+
--source-fps 50 \
|
| 156 |
+
--target-fps 30 \
|
| 157 |
+
--dof-order isaaclab
|
infer_render.sh
ADDED
|
@@ -0,0 +1,64 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
cd /cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/Psi0-yth-v18
|
| 2 |
+
source .venv-psi/bin/activate
|
| 3 |
+
[ -f .env ] && source .env
|
| 4 |
+
|
| 5 |
+
export PYTHONPATH=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/Psi0-yth-v18/src
|
| 6 |
+
export CUDA_VISIBLE_DEVICES=0
|
| 7 |
+
export PSI_INFER_FLOAT32=1
|
| 8 |
+
pkill -f monitor.py
|
| 9 |
+
|
| 10 |
+
# <modify_audio_infer> 推理脚本切换为音频输入,对齐当前audio-only + full-audio + 39D TextOp训练逻辑。
|
| 11 |
+
# 原逻辑:
|
| 12 |
+
# RUN_DIR="/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/Psi0-yth-v18/runs/posttrain_v1/posttrain.sg.flow1000.cwml.lr1.0e-04.b2048.gpus8.2604250959"
|
| 13 |
+
# OUTPUT_DIR="/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/Psi0-yth-v18/runs/posttrain_v1/2604250959"
|
| 14 |
+
# /cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/envs/Psi0/bin/python scripts/infer/seed_text_to_motion.py \
|
| 15 |
+
# --run-dir $RUN_DIR \
|
| 16 |
+
# --ckpt-step best \
|
| 17 |
+
# --output-dir $OUTPUT_DIR \
|
| 18 |
+
# --num-samples 1 \
|
| 19 |
+
# --start-index 23069 \
|
| 20 |
+
# --num-inference-steps 10 \
|
| 21 |
+
# --device cuda:0
|
| 22 |
+
|
| 23 |
+
RUN_TIMESTAMP=2604250426
|
| 24 |
+
RUN_PARENT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/Psi0-yth-v18/runs/posttrain_v1_audio_dance
|
| 25 |
+
RUN_DIR=$RUN_PARENT/posttrain.sag.flow1000.cwml.lr1.0e-04.b512.gpus8.$RUN_TIMESTAMP
|
| 26 |
+
OUTPUT_DIR=$RUN_PARENT/$RUN_TIMESTAMP.best
|
| 27 |
+
MOTION_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/GR00T-WholeBodyControl/gear_sonic_deploy/reference/converted/psi0_dance/gvhmr
|
| 28 |
+
VIDEO_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/dance_1723_video_psi0/dance_1723_video_10s
|
| 29 |
+
MOTION_NAME=movies_1mzl0Zh614A_0000
|
| 30 |
+
MOTION_DIR=$MOTION_ROOT/$MOTION_NAME
|
| 31 |
+
VIDEO_PATH=$VIDEO_ROOT/$MOTION_NAME.mp4
|
| 32 |
+
CKPT_STEP=best
|
| 33 |
+
PROMPT="Generate a dance sequence that matches the provided music."
|
| 34 |
+
|
| 35 |
+
rm -rf $OUTPUT_DIR
|
| 36 |
+
|
| 37 |
+
echo "Run dir: $RUN_DIR"
|
| 38 |
+
echo "Ckpt step: $CKPT_STEP"
|
| 39 |
+
echo "Prompt: $PROMPT"
|
| 40 |
+
echo "Motion dir: $MOTION_DIR"
|
| 41 |
+
echo "Video path: $VIDEO_PATH"
|
| 42 |
+
echo "Output dir: $OUTPUT_DIR"
|
| 43 |
+
|
| 44 |
+
python scripts/infer/seed_audio_to_motion.py \
|
| 45 |
+
--run-dir "$RUN_DIR" \
|
| 46 |
+
--ckpt-step "$CKPT_STEP" \
|
| 47 |
+
--output-dir "$OUTPUT_DIR" \
|
| 48 |
+
--motion-dir "$MOTION_DIR" \
|
| 49 |
+
--video-path "$VIDEO_PATH" \
|
| 50 |
+
--prompt "$PROMPT" \
|
| 51 |
+
--num-inference-steps 10 \
|
| 52 |
+
--device cuda:0
|
| 53 |
+
|
| 54 |
+
deactivate
|
| 55 |
+
unset PYTHONPATH
|
| 56 |
+
cd /cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/GR00T-WholeBodyControl/gear_sonic
|
| 57 |
+
conda activate /cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/envs/gmr
|
| 58 |
+
export PYOPENGL_PLATFORM="egl"
|
| 59 |
+
export MUJOCO_GL="egl"
|
| 60 |
+
xvfb-run -a python scripts/batch_render_reference_motion_mujoco.py \
|
| 61 |
+
--motion-dir "$OUTPUT_DIR" \
|
| 62 |
+
--source-fps 50 \
|
| 63 |
+
--target-fps 30 \
|
| 64 |
+
--dof-order isaaclab
|
infer_render_auto.sh
ADDED
|
@@ -0,0 +1,709 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#!/bin/bash
|
| 2 |
+
|
| 3 |
+
REPO_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/Psi0-yth-v20
|
| 4 |
+
DEFAULT_RUN_DIR=$REPO_ROOT/runs/posttrain_v1_trimodal/posttrain_actionhead_M.sg.flow1000.cwml.lr1.0e-04.b256.gpus8.2606241549
|
| 5 |
+
|
| 6 |
+
cd "$REPO_ROOT"
|
| 7 |
+
export PYTHONPATH="$REPO_ROOT/src"
|
| 8 |
+
|
| 9 |
+
source /cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/Psi0-yth-v15/.venv-psi/bin/activate
|
| 10 |
+
[ -f .env ] && source .env
|
| 11 |
+
|
| 12 |
+
export PYTHONPATH="$REPO_ROOT/src"
|
| 13 |
+
export CUDA_VISIBLE_DEVICES=${CUDA_VISIBLE_DEVICES:-0}
|
| 14 |
+
export PSI_INFER_FLOAT32=1
|
| 15 |
+
pkill -f monitor.py || true
|
| 16 |
+
|
| 17 |
+
canonical_mode() {
|
| 18 |
+
case "$1" in
|
| 19 |
+
all|multi|full)
|
| 20 |
+
echo all
|
| 21 |
+
;;
|
| 22 |
+
text|pure_text)
|
| 23 |
+
echo text
|
| 24 |
+
;;
|
| 25 |
+
audio|audio_text|audio_only|audio_pure)
|
| 26 |
+
echo audio
|
| 27 |
+
;;
|
| 28 |
+
audio_command|audio_command_text|audio_command_only|command_audio)
|
| 29 |
+
echo audio_command
|
| 30 |
+
;;
|
| 31 |
+
video_clip|video_clip_text|video_clip_only)
|
| 32 |
+
echo video_clip
|
| 33 |
+
;;
|
| 34 |
+
video|video_text|video_only)
|
| 35 |
+
echo video
|
| 36 |
+
;;
|
| 37 |
+
*)
|
| 38 |
+
echo "$1"
|
| 39 |
+
;;
|
| 40 |
+
esac
|
| 41 |
+
}
|
| 42 |
+
|
| 43 |
+
RAW_MODE=${PSI_AUTO_INFER_MODE:-all}
|
| 44 |
+
MODE=$(canonical_mode "$RAW_MODE")
|
| 45 |
+
MODE_SEQUENCE=(text audio audio_command video_clip)
|
| 46 |
+
|
| 47 |
+
RUNS_ROOT=${RUNS_ROOT:-$REPO_ROOT/runs/posttrain_v1_trimodal}
|
| 48 |
+
INFER_PY=${INFER_PY:-/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/Psi0-yth-v15/.venv-psi/bin/python}
|
| 49 |
+
RUN_DIR=${RUN_DIR:-auto}
|
| 50 |
+
|
| 51 |
+
if [ $# -ge 1 ]; then
|
| 52 |
+
RUN_DIR=$1
|
| 53 |
+
elif [ "$RUN_DIR" = "auto" ] || [ -z "$RUN_DIR" ]; then
|
| 54 |
+
AUTO_RUN_DIR=$("$INFER_PY" - "$MODE" "$RUNS_ROOT" <<'PY'
|
| 55 |
+
import json
|
| 56 |
+
import re
|
| 57 |
+
import sys
|
| 58 |
+
from pathlib import Path
|
| 59 |
+
|
| 60 |
+
mode = sys.argv[1]
|
| 61 |
+
runs_root = Path(sys.argv[2])
|
| 62 |
+
|
| 63 |
+
def ratio(data, key):
|
| 64 |
+
try:
|
| 65 |
+
return float(data.get(key, 0.0))
|
| 66 |
+
except Exception:
|
| 67 |
+
return 0.0
|
| 68 |
+
|
| 69 |
+
def supported_modes(run_config):
|
| 70 |
+
data = run_config.get("data", {})
|
| 71 |
+
modes = []
|
| 72 |
+
if ratio(data, "text_ratio") > 0 or "root_dir" in data:
|
| 73 |
+
modes.append("text")
|
| 74 |
+
if ratio(data, "audio_ratio") > 0 or ratio(data, "audio_only_ratio") > 0:
|
| 75 |
+
modes.append("audio")
|
| 76 |
+
if ratio(data, "audio_command_ratio") > 0 or ratio(data, "audio_command_only_ratio") > 0:
|
| 77 |
+
modes.append("audio_command")
|
| 78 |
+
if ratio(data, "video_clip_text_ratio") > 0 or ratio(data, "video_clip_only_ratio") > 0:
|
| 79 |
+
modes.append("video_clip")
|
| 80 |
+
if ratio(data, "video_ratio") > 0 or ratio(data, "video_only_ratio") > 0:
|
| 81 |
+
modes.append("video")
|
| 82 |
+
return modes
|
| 83 |
+
|
| 84 |
+
def run_sort_key(path):
|
| 85 |
+
match = re.search(r"(\d{10})$", path.name)
|
| 86 |
+
if match:
|
| 87 |
+
return int(match.group(1))
|
| 88 |
+
return int(path.stat().st_mtime)
|
| 89 |
+
|
| 90 |
+
candidates = []
|
| 91 |
+
for path in runs_root.glob("posttrain*.gpus8.*"):
|
| 92 |
+
config_path = path / "run_config.json"
|
| 93 |
+
if not config_path.exists():
|
| 94 |
+
continue
|
| 95 |
+
try:
|
| 96 |
+
run_config = json.loads(config_path.read_text())
|
| 97 |
+
except Exception:
|
| 98 |
+
continue
|
| 99 |
+
modes = supported_modes(run_config)
|
| 100 |
+
if mode == "all":
|
| 101 |
+
score = len([m for m in ("text", "audio", "audio_command", "video_clip", "video") if m in modes])
|
| 102 |
+
else:
|
| 103 |
+
score = 1 if mode in modes else 0
|
| 104 |
+
if score > 0:
|
| 105 |
+
candidates.append((score, run_sort_key(path), path))
|
| 106 |
+
|
| 107 |
+
if candidates:
|
| 108 |
+
candidates.sort()
|
| 109 |
+
print(str(candidates[-1][2]))
|
| 110 |
+
PY
|
| 111 |
+
)
|
| 112 |
+
if [ -n "$AUTO_RUN_DIR" ]; then
|
| 113 |
+
RUN_DIR=$AUTO_RUN_DIR
|
| 114 |
+
else
|
| 115 |
+
RUN_DIR=$DEFAULT_RUN_DIR
|
| 116 |
+
fi
|
| 117 |
+
fi
|
| 118 |
+
|
| 119 |
+
if [ -z "$RUN_DIR" ] || [ ! -f "$RUN_DIR/run_config.json" ]; then
|
| 120 |
+
echo "ERROR: cannot resolve a valid RUN_DIR for MODE=$MODE"
|
| 121 |
+
echo " Pass it explicitly: bash infer_render_auto.sh /path/to/run_dir"
|
| 122 |
+
exit 1
|
| 123 |
+
fi
|
| 124 |
+
|
| 125 |
+
SUPPORTED_MODES_CSV=$("$INFER_PY" - "$RUN_DIR" <<'PY'
|
| 126 |
+
import json
|
| 127 |
+
import sys
|
| 128 |
+
from pathlib import Path
|
| 129 |
+
|
| 130 |
+
run_dir = Path(sys.argv[1])
|
| 131 |
+
config = json.loads((run_dir / "run_config.json").read_text())
|
| 132 |
+
data = config.get("data", {})
|
| 133 |
+
|
| 134 |
+
def ratio(key):
|
| 135 |
+
try:
|
| 136 |
+
return float(data.get(key, 0.0))
|
| 137 |
+
except Exception:
|
| 138 |
+
return 0.0
|
| 139 |
+
|
| 140 |
+
supported = []
|
| 141 |
+
if ratio("text_ratio") > 0 or "root_dir" in data:
|
| 142 |
+
supported.append("text")
|
| 143 |
+
if ratio("audio_ratio") > 0 or ratio("audio_only_ratio") > 0:
|
| 144 |
+
supported.append("audio")
|
| 145 |
+
if ratio("audio_command_ratio") > 0 or ratio("audio_command_only_ratio") > 0:
|
| 146 |
+
supported.append("audio_command")
|
| 147 |
+
if ratio("video_clip_text_ratio") > 0 or ratio("video_clip_only_ratio") > 0:
|
| 148 |
+
supported.append("video_clip")
|
| 149 |
+
if ratio("video_ratio") > 0 or ratio("video_only_ratio") > 0:
|
| 150 |
+
supported.append("video")
|
| 151 |
+
print(",".join(supported))
|
| 152 |
+
PY
|
| 153 |
+
)
|
| 154 |
+
|
| 155 |
+
if [ -z "$SUPPORTED_MODES_CSV" ]; then
|
| 156 |
+
echo "ERROR: RUN_DIR does not expose any supported inference modes: $RUN_DIR"
|
| 157 |
+
exit 1
|
| 158 |
+
fi
|
| 159 |
+
|
| 160 |
+
IFS=',' read -r -a SUPPORTED_MODES <<< "$SUPPORTED_MODES_CSV"
|
| 161 |
+
|
| 162 |
+
mode_supported() {
|
| 163 |
+
local wanted
|
| 164 |
+
local supported
|
| 165 |
+
wanted=$(canonical_mode "$1")
|
| 166 |
+
for supported in "${SUPPORTED_MODES[@]}"; do
|
| 167 |
+
if [ "$supported" = "$wanted" ]; then
|
| 168 |
+
return 0
|
| 169 |
+
fi
|
| 170 |
+
done
|
| 171 |
+
return 1
|
| 172 |
+
}
|
| 173 |
+
|
| 174 |
+
TARGET_MODES=()
|
| 175 |
+
if [ "$MODE" = "all" ]; then
|
| 176 |
+
if mode_supported text; then
|
| 177 |
+
TARGET_MODES+=(text)
|
| 178 |
+
fi
|
| 179 |
+
if mode_supported audio; then
|
| 180 |
+
TARGET_MODES+=(audio)
|
| 181 |
+
fi
|
| 182 |
+
if mode_supported audio_command; then
|
| 183 |
+
TARGET_MODES+=(audio_command)
|
| 184 |
+
fi
|
| 185 |
+
if mode_supported video_clip; then
|
| 186 |
+
TARGET_MODES+=(video_clip)
|
| 187 |
+
elif mode_supported video; then
|
| 188 |
+
TARGET_MODES+=(video)
|
| 189 |
+
fi
|
| 190 |
+
if [ ${#TARGET_MODES[@]} -eq 0 ]; then
|
| 191 |
+
echo "ERROR: MODE=all but RUN_DIR has no supported target modes: $RUN_DIR"
|
| 192 |
+
exit 1
|
| 193 |
+
fi
|
| 194 |
+
else
|
| 195 |
+
if ! mode_supported "$RAW_MODE"; then
|
| 196 |
+
echo "ERROR: RUN_DIR does not look compatible with MODE=$RAW_MODE: $RUN_DIR"
|
| 197 |
+
echo " supported modes: ${SUPPORTED_MODES[*]}"
|
| 198 |
+
exit 1
|
| 199 |
+
fi
|
| 200 |
+
TARGET_MODES=("$RAW_MODE")
|
| 201 |
+
fi
|
| 202 |
+
|
| 203 |
+
CHECK_INTERVAL=10
|
| 204 |
+
NUM_SAMPLES=1
|
| 205 |
+
NUM_INFERENCE_STEPS=10
|
| 206 |
+
NUM_OUTPUT_FRAMES=500
|
| 207 |
+
AUDIO_NUM_OUTPUT_FRAMES=300
|
| 208 |
+
VIDEO_NUM_OUTPUT_FRAMES=300
|
| 209 |
+
VIDEO_CLIP_MOTION_FPS=50
|
| 210 |
+
DEVICE=cuda:0
|
| 211 |
+
GMR_PY=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/envs/gmr/bin/python
|
| 212 |
+
SONIC_RENDER_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/GR00T-WholeBodyControl/gear_sonic
|
| 213 |
+
SOURCE_FPS=50
|
| 214 |
+
TARGET_FPS=30
|
| 215 |
+
|
| 216 |
+
TEXT_PROMPTS=(
|
| 217 |
+
"A person jogs forward at a steady pace."
|
| 218 |
+
"A person walks forward calmly."
|
| 219 |
+
"A person performs a high jump and lands steadily."
|
| 220 |
+
"A person turns around in place."
|
| 221 |
+
)
|
| 222 |
+
TEXT_PROMPT_NAMES=(
|
| 223 |
+
jog_forward
|
| 224 |
+
walk_forward
|
| 225 |
+
high_jump
|
| 226 |
+
turn_in_place
|
| 227 |
+
)
|
| 228 |
+
TEXT_INIT_MOTION_DIRS=(
|
| 229 |
+
/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/Psi0/dataset/seed_all/g1/csv_sonic/230710/jog_ff_loop_360_R_001__A419_M
|
| 230 |
+
/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/Psi0/dataset/seed_all/g1/csv_sonic/230710/walk_ff_loop_315_R_001__A421_M
|
| 231 |
+
/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/Psi0/dataset/seed_all/g1/csv_sonic/230710/high_jump_R_001__A418_M
|
| 232 |
+
/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/Psi0/dataset/seed_all/g1/csv_sonic/230710/idle_turn_360_R_001__A418_M
|
| 233 |
+
)
|
| 234 |
+
|
| 235 |
+
VIDEO_MOTION_DIRS=(
|
| 236 |
+
/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/GR00T-WholeBodyControl/gear_sonic_deploy/reference/converted/psi0_dance/gvhmr/movies__F-SNBR4jhY_0007
|
| 237 |
+
)
|
| 238 |
+
VIDEO_PATHS=(
|
| 239 |
+
/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/dance_1723_video_psi0/dance_1723_video_10s/movies__F-SNBR4jhY_0007.mp4
|
| 240 |
+
)
|
| 241 |
+
VIDEO_PROMPTS=(
|
| 242 |
+
"Generate a motion sequence that matches the provided video."
|
| 243 |
+
)
|
| 244 |
+
|
| 245 |
+
AUDIO_MOTION_DIRS=(
|
| 246 |
+
/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/GR00T-WholeBodyControl/gear_sonic_deploy/reference/converted/psi0_dance/gvhmr/movies__F-SNBR4jhY_0007
|
| 247 |
+
)
|
| 248 |
+
AUDIO_PATHS=(
|
| 249 |
+
/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/dance_1723_video_psi0/dance_1723_video_10s/movies__F-SNBR4jhY_0007.mp4
|
| 250 |
+
)
|
| 251 |
+
AUDIO_PROMPTS=(
|
| 252 |
+
"Generate a dance sequence that matches the provided music."
|
| 253 |
+
)
|
| 254 |
+
|
| 255 |
+
AUDIO_COMMAND_MOTION_DIRS=(
|
| 256 |
+
/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/Psi0/dataset/seed_all/g1/csv_sonic/230710/walk_ff_loop_315_R_001__A421_M
|
| 257 |
+
)
|
| 258 |
+
AUDIO_COMMAND_PATHS=(
|
| 259 |
+
/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/qwen3-tts/seed_audio/000001/female_01_zh.wav
|
| 260 |
+
)
|
| 261 |
+
AUDIO_COMMAND_PROMPTS=(
|
| 262 |
+
"Generate a motion that follows the spoken instruction in the audio."
|
| 263 |
+
)
|
| 264 |
+
|
| 265 |
+
RUN_PARENT_DIR=$(dirname "$RUN_DIR")
|
| 266 |
+
RUN_NAME=$(basename "$RUN_DIR")
|
| 267 |
+
RUN_TAG=$(echo "$RUN_NAME" | awk -F. '{print $NF}')
|
| 268 |
+
|
| 269 |
+
if [ ${#TARGET_MODES[@]} -gt 1 ]; then
|
| 270 |
+
OUTPUT_ROOT=${PSI_AUTO_OUTPUT_DIR:-"$RUN_PARENT_DIR/$RUN_TAG.auto_infer_all"}
|
| 271 |
+
else
|
| 272 |
+
OUTPUT_ROOT=${PSI_AUTO_OUTPUT_DIR:-"$RUN_PARENT_DIR/$RUN_TAG.auto_infer.${TARGET_MODES[0]}"}
|
| 273 |
+
fi
|
| 274 |
+
|
| 275 |
+
output_dir_for_mode() {
|
| 276 |
+
local mode=$1
|
| 277 |
+
if [ ${#TARGET_MODES[@]} -gt 1 ]; then
|
| 278 |
+
echo "$OUTPUT_ROOT/$(canonical_mode "$mode")"
|
| 279 |
+
else
|
| 280 |
+
echo "$OUTPUT_ROOT"
|
| 281 |
+
fi
|
| 282 |
+
}
|
| 283 |
+
|
| 284 |
+
expected_samples_for_mode() {
|
| 285 |
+
case "$1" in
|
| 286 |
+
text|pure_text)
|
| 287 |
+
echo "${#TEXT_PROMPTS[@]}"
|
| 288 |
+
;;
|
| 289 |
+
audio|audio_text|audio_only|audio_pure)
|
| 290 |
+
echo "${#AUDIO_MOTION_DIRS[@]}"
|
| 291 |
+
;;
|
| 292 |
+
audio_command|audio_command_text|audio_command_only|command_audio)
|
| 293 |
+
echo "${#AUDIO_COMMAND_MOTION_DIRS[@]}"
|
| 294 |
+
;;
|
| 295 |
+
video_clip|video_clip_text|video_clip_only|video|video_text|video_only)
|
| 296 |
+
echo "${#VIDEO_MOTION_DIRS[@]}"
|
| 297 |
+
;;
|
| 298 |
+
*)
|
| 299 |
+
echo 0
|
| 300 |
+
;;
|
| 301 |
+
esac
|
| 302 |
+
}
|
| 303 |
+
|
| 304 |
+
expected_motion_frames_for_mode() {
|
| 305 |
+
case "$1" in
|
| 306 |
+
text|pure_text)
|
| 307 |
+
echo "$NUM_OUTPUT_FRAMES"
|
| 308 |
+
;;
|
| 309 |
+
audio|audio_text|audio_only|audio_pure|audio_command|audio_command_text|audio_command_only|command_audio)
|
| 310 |
+
echo "$AUDIO_NUM_OUTPUT_FRAMES"
|
| 311 |
+
;;
|
| 312 |
+
video_clip|video_clip_text|video_clip_only|video|video_text|video_only)
|
| 313 |
+
echo "$VIDEO_NUM_OUTPUT_FRAMES"
|
| 314 |
+
;;
|
| 315 |
+
*)
|
| 316 |
+
echo "$NUM_OUTPUT_FRAMES"
|
| 317 |
+
;;
|
| 318 |
+
esac
|
| 319 |
+
}
|
| 320 |
+
|
| 321 |
+
min_render_frames_for_mode() {
|
| 322 |
+
local mode=$1
|
| 323 |
+
local motion_frames
|
| 324 |
+
motion_frames=$(expected_motion_frames_for_mode "$mode")
|
| 325 |
+
python -c 'import sys
|
| 326 |
+
motion_frames = int(sys.argv[1])
|
| 327 |
+
source_fps = float(sys.argv[2])
|
| 328 |
+
target_fps = float(sys.argv[3])
|
| 329 |
+
print(int(round((max(motion_frames, 1) - 1) / source_fps * target_fps)) + 1)' "$motion_frames" "$SOURCE_FPS" "$TARGET_FPS"
|
| 330 |
+
}
|
| 331 |
+
|
| 332 |
+
count_valid_render_mp4() {
|
| 333 |
+
local step_dir=$1
|
| 334 |
+
local min_render_frames=$2
|
| 335 |
+
local valid_render_mp4_count=0
|
| 336 |
+
local mp4_path
|
| 337 |
+
local mp4_frames
|
| 338 |
+
|
| 339 |
+
while IFS= read -r mp4_path; do
|
| 340 |
+
mp4_frames=$(ffprobe -v error -select_streams v:0 -show_entries stream=nb_frames -of default=noprint_wrappers=1:nokey=1 "$mp4_path" 2>/dev/null | head -n1)
|
| 341 |
+
if [ -n "$mp4_frames" ] && [ "$mp4_frames" -ge "$min_render_frames" ] 2>/dev/null; then
|
| 342 |
+
valid_render_mp4_count=$((valid_render_mp4_count + 1))
|
| 343 |
+
fi
|
| 344 |
+
done < <(find "$step_dir" -name 'reference_motion.mp4' -print 2>/dev/null)
|
| 345 |
+
|
| 346 |
+
echo "$valid_render_mp4_count"
|
| 347 |
+
}
|
| 348 |
+
|
| 349 |
+
run_infer_for_mode() {
|
| 350 |
+
local mode=$1
|
| 351 |
+
local ckpt_name=$2
|
| 352 |
+
local ckpt_step=$3
|
| 353 |
+
local step_dir=$4
|
| 354 |
+
local mode_key
|
| 355 |
+
local sample_i
|
| 356 |
+
local PROMPT
|
| 357 |
+
local PROMPT_NAME
|
| 358 |
+
local INIT_MOTION_DIR
|
| 359 |
+
local MOTION_DIR
|
| 360 |
+
local AUDIO_PATH
|
| 361 |
+
local VIDEO_PATH
|
| 362 |
+
local AUDIO_CONFIG_KEY
|
| 363 |
+
local AUDIO_MODE_NAME
|
| 364 |
+
local infer_failed=0
|
| 365 |
+
|
| 366 |
+
mode_key=$(canonical_mode "$mode")
|
| 367 |
+
|
| 368 |
+
case "$mode" in
|
| 369 |
+
text|pure_text)
|
| 370 |
+
for sample_i in "${!TEXT_PROMPTS[@]}"; do
|
| 371 |
+
PROMPT=${TEXT_PROMPTS[$sample_i]}
|
| 372 |
+
PROMPT_NAME=${TEXT_PROMPT_NAMES[$sample_i]}
|
| 373 |
+
INIT_MOTION_DIR=${TEXT_INIT_MOTION_DIRS[$sample_i]}
|
| 374 |
+
echo " [推理] 开始 $mode infer sample=$sample_i"
|
| 375 |
+
echo " prompt: $PROMPT"
|
| 376 |
+
echo " init : $INIT_MOTION_DIR"
|
| 377 |
+
"$INFER_PY" scripts/infer/seed_text_to_motion.py \
|
| 378 |
+
--run-dir "$RUN_DIR" \
|
| 379 |
+
--ckpt-step "$ckpt_step" \
|
| 380 |
+
--output-dir "$step_dir" \
|
| 381 |
+
--prompt "$PROMPT" \
|
| 382 |
+
--prompt-motion-name "$PROMPT_NAME" \
|
| 383 |
+
--init-motion-dir "$INIT_MOTION_DIR" \
|
| 384 |
+
--num-inference-steps "$NUM_INFERENCE_STEPS" \
|
| 385 |
+
--num-output-frames "$NUM_OUTPUT_FRAMES" \
|
| 386 |
+
--force-text-only \
|
| 387 |
+
--device "$DEVICE"
|
| 388 |
+
if [ $? -ne 0 ]; then
|
| 389 |
+
echo " [错误] $ckpt_name $mode infer失败,sample=$sample_i"
|
| 390 |
+
infer_failed=1
|
| 391 |
+
break
|
| 392 |
+
fi
|
| 393 |
+
done
|
| 394 |
+
;;
|
| 395 |
+
audio|audio_text|audio_only|audio_pure|audio_command|audio_command_text|audio_command_only|command_audio)
|
| 396 |
+
if [ "$mode_key" = "audio" ]; then
|
| 397 |
+
AUDIO_CONFIG_KEY=audio
|
| 398 |
+
AUDIO_MODE_NAME=audio
|
| 399 |
+
MOTION_DIRS=("${AUDIO_MOTION_DIRS[@]}")
|
| 400 |
+
AUDIO_FILES=("${AUDIO_PATHS[@]}")
|
| 401 |
+
PROMPTS=("${AUDIO_PROMPTS[@]}")
|
| 402 |
+
else
|
| 403 |
+
AUDIO_CONFIG_KEY=audio_command
|
| 404 |
+
AUDIO_MODE_NAME=audio_command
|
| 405 |
+
MOTION_DIRS=("${AUDIO_COMMAND_MOTION_DIRS[@]}")
|
| 406 |
+
AUDIO_FILES=("${AUDIO_COMMAND_PATHS[@]}")
|
| 407 |
+
PROMPTS=("${AUDIO_COMMAND_PROMPTS[@]}")
|
| 408 |
+
fi
|
| 409 |
+
|
| 410 |
+
for sample_i in "${!MOTION_DIRS[@]}"; do
|
| 411 |
+
MOTION_DIR=${MOTION_DIRS[$sample_i]}
|
| 412 |
+
AUDIO_PATH=${AUDIO_FILES[$sample_i]}
|
| 413 |
+
PROMPT=${PROMPTS[$sample_i]}
|
| 414 |
+
|
| 415 |
+
echo " [推理] 开始 $mode infer sample=$sample_i"
|
| 416 |
+
echo " audio_config: $AUDIO_CONFIG_KEY"
|
| 417 |
+
echo " motion : $MOTION_DIR"
|
| 418 |
+
echo " audio : $AUDIO_PATH"
|
| 419 |
+
"$INFER_PY" scripts/infer/seed_audio_to_motion.py \
|
| 420 |
+
--run-dir "$RUN_DIR" \
|
| 421 |
+
--ckpt-step "$ckpt_step" \
|
| 422 |
+
--output-dir "$step_dir" \
|
| 423 |
+
--motion-dir "$MOTION_DIR" \
|
| 424 |
+
--video-path "$AUDIO_PATH" \
|
| 425 |
+
--audio-config-key "$AUDIO_CONFIG_KEY" \
|
| 426 |
+
--prompt "$PROMPT" \
|
| 427 |
+
--num-inference-steps "$NUM_INFERENCE_STEPS" \
|
| 428 |
+
--num-output-frames "$AUDIO_NUM_OUTPUT_FRAMES" \
|
| 429 |
+
--device "$DEVICE"
|
| 430 |
+
if [ $? -ne 0 ]; then
|
| 431 |
+
echo " [错误] $ckpt_name $AUDIO_MODE_NAME infer失败,sample=$sample_i"
|
| 432 |
+
infer_failed=1
|
| 433 |
+
break
|
| 434 |
+
fi
|
| 435 |
+
done
|
| 436 |
+
;;
|
| 437 |
+
video_clip|video_clip_text|video_clip_only|video|video_text|video_only)
|
| 438 |
+
for sample_i in "${!VIDEO_MOTION_DIRS[@]}"; do
|
| 439 |
+
MOTION_DIR=${VIDEO_MOTION_DIRS[$sample_i]}
|
| 440 |
+
VIDEO_PATH=${VIDEO_PATHS[$sample_i]}
|
| 441 |
+
PROMPT=${VIDEO_PROMPTS[$sample_i]}
|
| 442 |
+
VIDEO_ARGS=(--video-config-key video)
|
| 443 |
+
|
| 444 |
+
if [ "$mode_key" = "video_clip" ]; then
|
| 445 |
+
VIDEO_ARGS=(--video-config-key video_clip --video-clip --video-clip-motion-fps "$VIDEO_CLIP_MOTION_FPS")
|
| 446 |
+
fi
|
| 447 |
+
if [ "$mode" = "video_only" ] || [ "$mode" = "video_clip_only" ]; then
|
| 448 |
+
PROMPT=""
|
| 449 |
+
fi
|
| 450 |
+
|
| 451 |
+
echo " [推理] 开始 $mode infer sample=$sample_i"
|
| 452 |
+
echo " motion: $MOTION_DIR"
|
| 453 |
+
echo " video : $VIDEO_PATH"
|
| 454 |
+
"$INFER_PY" scripts/infer/seed_video_to_motion.py \
|
| 455 |
+
--run-dir "$RUN_DIR" \
|
| 456 |
+
--ckpt-step "$ckpt_step" \
|
| 457 |
+
--output-dir "$step_dir" \
|
| 458 |
+
--motion-dir "$MOTION_DIR" \
|
| 459 |
+
--video-path "$VIDEO_PATH" \
|
| 460 |
+
--prompt "$PROMPT" \
|
| 461 |
+
--num-inference-steps "$NUM_INFERENCE_STEPS" \
|
| 462 |
+
--num-output-frames "$VIDEO_NUM_OUTPUT_FRAMES" \
|
| 463 |
+
"${VIDEO_ARGS[@]}" \
|
| 464 |
+
--device "$DEVICE"
|
| 465 |
+
if [ $? -ne 0 ]; then
|
| 466 |
+
echo " [错误] $ckpt_name $mode infer失败,sample=$sample_i"
|
| 467 |
+
infer_failed=1
|
| 468 |
+
break
|
| 469 |
+
fi
|
| 470 |
+
done
|
| 471 |
+
;;
|
| 472 |
+
*)
|
| 473 |
+
echo " [错误] unknown mode: $mode"
|
| 474 |
+
infer_failed=1
|
| 475 |
+
;;
|
| 476 |
+
esac
|
| 477 |
+
|
| 478 |
+
if [ "$infer_failed" -ne 0 ]; then
|
| 479 |
+
return 1
|
| 480 |
+
fi
|
| 481 |
+
return 0
|
| 482 |
+
}
|
| 483 |
+
|
| 484 |
+
run_render_for_mode() {
|
| 485 |
+
local mode=$1
|
| 486 |
+
local ckpt_name=$2
|
| 487 |
+
local step_dir=$3
|
| 488 |
+
local expected_samples=$4
|
| 489 |
+
local min_render_frames=$5
|
| 490 |
+
local valid_render_mp4_count
|
| 491 |
+
|
| 492 |
+
echo " [渲染] 开始 $mode render"
|
| 493 |
+
(
|
| 494 |
+
cd "$SONIC_RENDER_ROOT"
|
| 495 |
+
if [ -n "$XVFB_RUN" ]; then
|
| 496 |
+
env -u PYTHONPATH \
|
| 497 |
+
PYOPENGL_PLATFORM=egl \
|
| 498 |
+
MUJOCO_GL=egl \
|
| 499 |
+
QT_X11_NO_MITSHM=1 \
|
| 500 |
+
SONIC_RENDER_HEADLESS_CAMERA=1 \
|
| 501 |
+
SONIC_RENDER_LOCAL_TMP_OUTPUT=1 \
|
| 502 |
+
SONIC_RENDER_TMPDIR=/tmp \
|
| 503 |
+
"$XVFB_RUN" -a -s "-screen 0 1280x1024x24 -extension MIT-SHM" \
|
| 504 |
+
"$GMR_PY" scripts/batch_render_reference_motion_mujoco.py \
|
| 505 |
+
--motion-dir "$step_dir" \
|
| 506 |
+
--source-fps "$SOURCE_FPS" \
|
| 507 |
+
--target-fps "$TARGET_FPS" \
|
| 508 |
+
--dof-order isaaclab
|
| 509 |
+
else
|
| 510 |
+
echo " [渲染] xvfb-run not found; use headless EGL camera"
|
| 511 |
+
env -u PYTHONPATH \
|
| 512 |
+
PYOPENGL_PLATFORM=egl \
|
| 513 |
+
MUJOCO_GL=egl \
|
| 514 |
+
QT_X11_NO_MITSHM=1 \
|
| 515 |
+
SONIC_RENDER_HEADLESS_CAMERA=1 \
|
| 516 |
+
SONIC_RENDER_LOCAL_TMP_OUTPUT=1 \
|
| 517 |
+
SONIC_RENDER_TMPDIR=/tmp \
|
| 518 |
+
"$GMR_PY" scripts/batch_render_reference_motion_mujoco.py \
|
| 519 |
+
--motion-dir "$step_dir" \
|
| 520 |
+
--source-fps "$SOURCE_FPS" \
|
| 521 |
+
--target-fps "$TARGET_FPS" \
|
| 522 |
+
--dof-order isaaclab
|
| 523 |
+
fi
|
| 524 |
+
)
|
| 525 |
+
|
| 526 |
+
if [ $? -ne 0 ]; then
|
| 527 |
+
valid_render_mp4_count=$(count_valid_render_mp4 "$step_dir" "$min_render_frames")
|
| 528 |
+
if [ "$valid_render_mp4_count" -ge "$expected_samples" ]; then
|
| 529 |
+
echo " [警告] $mode render返回非0,但已生成 ${valid_render_mp4_count}/${expected_samples} 个有效reference_motion.mp4,视为完成"
|
| 530 |
+
else
|
| 531 |
+
echo " [错误] $ckpt_name $mode render失败,当前有效reference_motion.mp4数量: $valid_render_mp4_count/${expected_samples}"
|
| 532 |
+
return 1
|
| 533 |
+
fi
|
| 534 |
+
else
|
| 535 |
+
echo " [完成] $ckpt_name $mode render成功"
|
| 536 |
+
fi
|
| 537 |
+
return 0
|
| 538 |
+
}
|
| 539 |
+
|
| 540 |
+
process_mode_for_ckpt() {
|
| 541 |
+
local mode=$1
|
| 542 |
+
local ckpt_name=$2
|
| 543 |
+
local ckpt_step=$3
|
| 544 |
+
local output_dir
|
| 545 |
+
local step_dir
|
| 546 |
+
local expected_samples
|
| 547 |
+
local infer_csv_count
|
| 548 |
+
local min_render_frames
|
| 549 |
+
local valid_render_mp4_count
|
| 550 |
+
|
| 551 |
+
output_dir=$(output_dir_for_mode "$mode")
|
| 552 |
+
step_dir="$output_dir/step_$ckpt_step"
|
| 553 |
+
expected_samples=$(expected_samples_for_mode "$mode")
|
| 554 |
+
min_render_frames=$(min_render_frames_for_mode "$mode")
|
| 555 |
+
|
| 556 |
+
mkdir -p "$step_dir"
|
| 557 |
+
|
| 558 |
+
infer_csv_count=$(find "$step_dir" -name 'body_pos.csv' -print 2>/dev/null | wc -l | tr -d ' ')
|
| 559 |
+
valid_render_mp4_count=$(count_valid_render_mp4 "$step_dir" "$min_render_frames")
|
| 560 |
+
|
| 561 |
+
echo " [模式] $mode"
|
| 562 |
+
echo " 输出目录: $step_dir"
|
| 563 |
+
echo " infer: ${infer_csv_count}/${expected_samples}"
|
| 564 |
+
echo " render: ${valid_render_mp4_count}/${expected_samples}"
|
| 565 |
+
|
| 566 |
+
if [ "$infer_csv_count" -lt "$expected_samples" ]; then
|
| 567 |
+
if ! run_infer_for_mode "$mode" "$ckpt_name" "$ckpt_step" "$step_dir"; then
|
| 568 |
+
return 1
|
| 569 |
+
fi
|
| 570 |
+
infer_csv_count=$(find "$step_dir" -name 'body_pos.csv' -print 2>/dev/null | wc -l | tr -d ' ')
|
| 571 |
+
echo " [完成] $ckpt_name $mode infer成功,当前body_pos.csv数量: $infer_csv_count"
|
| 572 |
+
else
|
| 573 |
+
echo " [跳过] $mode infer已完成: ${infer_csv_count}/${expected_samples} 个body_pos.csv"
|
| 574 |
+
fi
|
| 575 |
+
|
| 576 |
+
valid_render_mp4_count=$(count_valid_render_mp4 "$step_dir" "$min_render_frames")
|
| 577 |
+
if [ "$valid_render_mp4_count" -lt "$expected_samples" ]; then
|
| 578 |
+
if ! run_render_for_mode "$mode" "$ckpt_name" "$step_dir" "$expected_samples" "$min_render_frames"; then
|
| 579 |
+
return 1
|
| 580 |
+
fi
|
| 581 |
+
else
|
| 582 |
+
echo " [跳过] $mode render已完成: ${valid_render_mp4_count}/${expected_samples} 个有效reference_motion.mp4"
|
| 583 |
+
fi
|
| 584 |
+
|
| 585 |
+
return 0
|
| 586 |
+
}
|
| 587 |
+
|
| 588 |
+
if [ -x /usr/bin/xvfb-run ]; then
|
| 589 |
+
XVFB_RUN=/usr/bin/xvfb-run
|
| 590 |
+
elif [ -x /bin/xvfb-run ]; then
|
| 591 |
+
XVFB_RUN=/bin/xvfb-run
|
| 592 |
+
elif command -v xvfb-run >/dev/null 2>&1; then
|
| 593 |
+
XVFB_RUN=$(command -v xvfb-run)
|
| 594 |
+
else
|
| 595 |
+
XVFB_RUN=
|
| 596 |
+
fi
|
| 597 |
+
|
| 598 |
+
export PYOPENGL_PLATFORM=egl
|
| 599 |
+
export MUJOCO_GL=egl
|
| 600 |
+
export QT_X11_NO_MITSHM=1
|
| 601 |
+
|
| 602 |
+
echo "=========================================="
|
| 603 |
+
echo "Auto Infer + Render Monitor"
|
| 604 |
+
echo "=========================================="
|
| 605 |
+
echo "RUN_DIR: $RUN_DIR"
|
| 606 |
+
echo "OUTPUT_ROOT: $OUTPUT_ROOT"
|
| 607 |
+
echo "RAW_MODE: $RAW_MODE"
|
| 608 |
+
echo "MODE: $MODE"
|
| 609 |
+
echo "SUPPORTED_MODES: ${SUPPORTED_MODES[*]}"
|
| 610 |
+
echo "TARGET_MODES: ${TARGET_MODES[*]}"
|
| 611 |
+
echo "CHECK_INTERVAL: ${CHECK_INTERVAL}s"
|
| 612 |
+
echo "NUM_OUTPUT_FRAMES: $NUM_OUTPUT_FRAMES"
|
| 613 |
+
echo "AUDIO_NUM_OUTPUT_FRAMES: $AUDIO_NUM_OUTPUT_FRAMES"
|
| 614 |
+
echo "VIDEO_NUM_OUTPUT_FRAMES: $VIDEO_NUM_OUTPUT_FRAMES"
|
| 615 |
+
echo "NUM_INFERENCE_STEPS: $NUM_INFERENCE_STEPS"
|
| 616 |
+
echo "XVFB_RUN: ${XVFB_RUN:-<not found, use headless EGL camera>}"
|
| 617 |
+
echo "NOTE: default MODE=all. Set PSI_AUTO_INFER_MODE to run a single modality."
|
| 618 |
+
echo "NOTE: single-mode values: text / audio / audio_command / video_clip"
|
| 619 |
+
echo "=========================================="
|
| 620 |
+
|
| 621 |
+
mkdir -p "$OUTPUT_ROOT"
|
| 622 |
+
for mode in "${TARGET_MODES[@]}"; do
|
| 623 |
+
mode_output_dir=$(output_dir_for_mode "$mode")
|
| 624 |
+
mkdir -p "$mode_output_dir"
|
| 625 |
+
echo "OUTPUT_DIR[$mode]: $mode_output_dir"
|
| 626 |
+
done
|
| 627 |
+
|
| 628 |
+
while true; do
|
| 629 |
+
echo ""
|
| 630 |
+
echo "[$(date '+%Y-%m-%d %H:%M:%S')] 扫描新的checkpoint..."
|
| 631 |
+
|
| 632 |
+
pending_ckpts=()
|
| 633 |
+
|
| 634 |
+
checkpoint_dirs=()
|
| 635 |
+
for ckpt_dir in "$RUN_DIR"/checkpoints/ckpt_*; do
|
| 636 |
+
[ -d "$ckpt_dir" ] || continue
|
| 637 |
+
checkpoint_dirs+=("$ckpt_dir")
|
| 638 |
+
done
|
| 639 |
+
if [ ${#checkpoint_dirs[@]} -eq 0 ] && [ -d "$RUN_DIR/checkpoints/best_ckpt" ]; then
|
| 640 |
+
checkpoint_dirs+=("$RUN_DIR/checkpoints/best_ckpt")
|
| 641 |
+
fi
|
| 642 |
+
|
| 643 |
+
for ckpt_dir in "${checkpoint_dirs[@]}"; do
|
| 644 |
+
ckpt_name=$(basename "$ckpt_dir")
|
| 645 |
+
if [ "$ckpt_name" = "best_ckpt" ]; then
|
| 646 |
+
ckpt_step=best
|
| 647 |
+
sort_step=999999999999
|
| 648 |
+
elif [[ "$ckpt_name" =~ ckpt_([0-9]+) ]]; then
|
| 649 |
+
ckpt_step="${BASH_REMATCH[1]}"
|
| 650 |
+
sort_step="$ckpt_step"
|
| 651 |
+
else
|
| 652 |
+
echo " [警告] 无法从 $ckpt_name 提取step数,跳过"
|
| 653 |
+
continue
|
| 654 |
+
fi
|
| 655 |
+
|
| 656 |
+
ckpt_pending_modes=()
|
| 657 |
+
for scan_mode in "${TARGET_MODES[@]}"; do
|
| 658 |
+
scan_step_dir="$(output_dir_for_mode "$scan_mode")/step_$ckpt_step"
|
| 659 |
+
scan_expected_samples=$(expected_samples_for_mode "$scan_mode")
|
| 660 |
+
scan_min_render_frames=$(min_render_frames_for_mode "$scan_mode")
|
| 661 |
+
scan_valid_render_mp4_count=$(count_valid_render_mp4 "$scan_step_dir" "$scan_min_render_frames")
|
| 662 |
+
if [ "$scan_valid_render_mp4_count" -lt "$scan_expected_samples" ]; then
|
| 663 |
+
ckpt_pending_modes+=("$scan_mode")
|
| 664 |
+
fi
|
| 665 |
+
done
|
| 666 |
+
|
| 667 |
+
if [ ${#ckpt_pending_modes[@]} -eq 0 ]; then
|
| 668 |
+
echo " [跳过] $ckpt_name (所有目标模态都已完成)"
|
| 669 |
+
continue
|
| 670 |
+
fi
|
| 671 |
+
|
| 672 |
+
pending_modes_csv=$(IFS=,; echo "${ckpt_pending_modes[*]}")
|
| 673 |
+
pending_ckpts+=("${sort_step}|${ckpt_step}|${ckpt_dir}|${pending_modes_csv}")
|
| 674 |
+
done
|
| 675 |
+
|
| 676 |
+
if [ ${#pending_ckpts[@]} -eq 0 ]; then
|
| 677 |
+
echo " [无] 没有待处理的checkpoint"
|
| 678 |
+
echo "[$(date '+%Y-%m-%d %H:%M:%S')] 等待 ${CHECK_INTERVAL} 秒后再次检查..."
|
| 679 |
+
sleep "$CHECK_INTERVAL"
|
| 680 |
+
continue
|
| 681 |
+
fi
|
| 682 |
+
|
| 683 |
+
best_item=$(printf '%s\n' "${pending_ckpts[@]}" | sort -t'|' -k1,1nr | head -n1)
|
| 684 |
+
IFS='|' read -r sort_step ckpt_step ckpt_dir pending_modes_csv <<< "$best_item"
|
| 685 |
+
ckpt_name=$(basename "$ckpt_dir")
|
| 686 |
+
IFS=',' read -r -a CKPT_PENDING_MODES <<< "$pending_modes_csv"
|
| 687 |
+
|
| 688 |
+
echo " [发现] 待处理checkpoint共 ${#pending_ckpts[@]} 个,选择step最大的:"
|
| 689 |
+
echo " $ckpt_name"
|
| 690 |
+
echo " [步骤] step: $ckpt_step"
|
| 691 |
+
echo " [模式] 待处理: ${CKPT_PENDING_MODES[*]}"
|
| 692 |
+
|
| 693 |
+
mode_failed=0
|
| 694 |
+
for mode in "${CKPT_PENDING_MODES[@]}"; do
|
| 695 |
+
if ! process_mode_for_ckpt "$mode" "$ckpt_name" "$ckpt_step"; then
|
| 696 |
+
mode_failed=1
|
| 697 |
+
break
|
| 698 |
+
fi
|
| 699 |
+
done
|
| 700 |
+
|
| 701 |
+
if [ "$mode_failed" -ne 0 ]; then
|
| 702 |
+
echo "[$(date '+%Y-%m-%d %H:%M:%S')] 等待 ${CHECK_INTERVAL} 秒后再次检查..."
|
| 703 |
+
sleep "$CHECK_INTERVAL"
|
| 704 |
+
continue
|
| 705 |
+
fi
|
| 706 |
+
|
| 707 |
+
echo "[$(date '+%Y-%m-%d %H:%M:%S')] 等待 ${CHECK_INTERVAL} 秒后再次检查..."
|
| 708 |
+
sleep "$CHECK_INTERVAL"
|
| 709 |
+
done
|
plan_0.md
ADDED
|
@@ -0,0 +1,374 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Psi0 改造成 Text-to-Humanoid-Motion 计划(最小改动版)
|
| 2 |
+
|
| 3 |
+
## 0) 目标与约束
|
| 4 |
+
|
| 5 |
+
你的目标可以抽象成:
|
| 6 |
+
|
| 7 |
+
1. 输入:文本 caption(先不使用图像/音频)。
|
| 8 |
+
2. 输出:机器人动作序列(你定义为每帧 36 维,`29 dof + xyz + wxyz`)。
|
| 9 |
+
3. 训练策略:不走 pretrain,直接用 posttrain 框架,冻结 VLM,只训练 action expert(扩散动作头)。
|
| 10 |
+
4. 未来扩展:VLM 换到 `Qwen-2.5-Omni-3B`,当前先做 text-only smoke test。
|
| 11 |
+
5. 工程要求:尽可能少改现有代码。
|
| 12 |
+
|
| 13 |
+
下面的方案按这个目标严格展开。
|
| 14 |
+
|
| 15 |
+
---
|
| 16 |
+
|
| 17 |
+
## 1) 建议先看的代码(阅读顺序)
|
| 18 |
+
|
| 19 |
+
按这个顺序看,最快建立全局心智模型:
|
| 20 |
+
|
| 21 |
+
1. 训练主入口与配置注入
|
| 22 |
+
- `scripts/train.py`(训练循环、配置解析入口)
|
| 23 |
+
- `src/psi/config/config.py`(`LaunchConfig/TrainConfig/DataConfig/ModelConfig`)
|
| 24 |
+
- `src/psi/config/train/posttrain_he_psi0_config.py`
|
| 25 |
+
- `src/psi/config/train/posttrain_mix_psi0_config.py`
|
| 26 |
+
- `scripts/train/psi0/posttrain-he-psi0.sh`
|
| 27 |
+
|
| 28 |
+
2. posttrain trainer 主流程
|
| 29 |
+
- `src/psi/trainers/posttrain.py`
|
| 30 |
+
- `src/psi/trainers/trainer.py`
|
| 31 |
+
|
| 32 |
+
3. 数据层(raw -> transform -> batch)
|
| 33 |
+
- `src/psi/config/data_he.py`
|
| 34 |
+
- `src/psi/data/humanoid/he_raw_dataset.py`
|
| 35 |
+
- `src/psi/data/dataset.py`
|
| 36 |
+
- `src/psi/config/transform.py`(重点:`HEPosttrainRepackTransform`, `ActionStateTransform`, `Psi0ModelTransform`)
|
| 37 |
+
- `src/psi/trainers/qwen3vl_mixin.py`(`PaddedCollatorForTogether`)
|
| 38 |
+
|
| 39 |
+
4. 模型层(VLM + action expert)
|
| 40 |
+
- `src/psi/config/model_psi0.py`
|
| 41 |
+
- `src/psi/models/psi0.py`(重点:`Psi0Model.forward`, `ActionTransformerModel`, `ObservationProjection`)
|
| 42 |
+
|
| 43 |
+
5. 数据预处理脚本(参考)
|
| 44 |
+
- `scripts/data/raw_he_to_psi0.py`
|
| 45 |
+
- `scripts/data/raw_to_lerobot.py`
|
| 46 |
+
- `scripts/data/calc_modality_stats.py`
|
| 47 |
+
- `scripts/data/raw_dexmate_psi0.py`
|
| 48 |
+
|
| 49 |
+
---
|
| 50 |
+
|
| 51 |
+
## 2) 当前 posttrain 数据流和训练流(从数据处理开始)
|
| 52 |
+
|
| 53 |
+
## 2.1 训练入口与实例化
|
| 54 |
+
|
| 55 |
+
1. `scripts/train.py` 根据命令行第一个参数动态导入 config 模块(例如 `posttrain_he_psi0_config`)。
|
| 56 |
+
2. `Trainer.instantiate` 会按 `cfg.train.name` 选择 `psi.trainers.posttrain.PosttrainTrainer`。
|
| 57 |
+
3. 流程是固定的:`init_models -> create_datasets -> create_dataloaders -> create_optimizer_and_scheduler -> prepare -> training loop`。
|
| 58 |
+
|
| 59 |
+
对应代码:
|
| 60 |
+
- `scripts/train.py:169-219, 233-314, 350-360`
|
| 61 |
+
|
| 62 |
+
## 2.2 数据集原始样本(HE 例子)
|
| 63 |
+
|
| 64 |
+
`HERawDataset.__getitem__` 输出 raw dict,核心字段:
|
| 65 |
+
|
| 66 |
+
1. `observation.images.egocentric`: `(To, 3, H, W)`,默认 `To=1`。
|
| 67 |
+
2. `observation.hand_joints`, `observation.arm_joints`。
|
| 68 |
+
3. 展平后的 action 字段(例如 `action.joint_angles`)。
|
| 69 |
+
4. `task`(instruction 文本),`action_mask`,`obs_mask`。
|
| 70 |
+
|
| 71 |
+
对应代码:
|
| 72 |
+
- `src/psi/data/humanoid/he_raw_dataset.py:176-237`
|
| 73 |
+
|
| 74 |
+
## 2.3 三段 transform:repack -> field -> model
|
| 75 |
+
|
| 76 |
+
`MapStyleDataset.__getitem__` 会执行:
|
| 77 |
+
|
| 78 |
+
1. `repack`:把 raw 字段整理成统一结构。
|
| 79 |
+
2. `field`:做动作/状态归一化、padding、反归一化逻辑。
|
| 80 |
+
3. `model`:把图像+文本喂给 processor,生成 `input_ids/pixel_values/...`。
|
| 81 |
+
|
| 82 |
+
对应代码:
|
| 83 |
+
- `src/psi/data/dataset.py:23-26`
|
| 84 |
+
- `src/psi/config/transform.py`(`DataTransform.__call__`)
|
| 85 |
+
|
| 86 |
+
### repack(posttrain HE)
|
| 87 |
+
|
| 88 |
+
`HEPosttrainRepackTransform` 做的事:
|
| 89 |
+
|
| 90 |
+
1. 构造 `states`、`actions`。
|
| 91 |
+
2. 如果 `use_delta_actions=True`,将动作转成 delta(`actions[1:] - actions[:-1]`)。
|
| 92 |
+
3. 按 `pad_action_dim/pad_state_dim` 补齐。
|
| 93 |
+
4. 输出统一字段:`observations`, `states`, `actions`, `instruction`, `actions_mask` 等。
|
| 94 |
+
|
| 95 |
+
对应代码:
|
| 96 |
+
- `src/psi/config/transform.py:705-767`
|
| 97 |
+
|
| 98 |
+
### field(动作归一化)
|
| 99 |
+
|
| 100 |
+
`ActionStateTransform`:
|
| 101 |
+
|
| 102 |
+
1. 从 `stat_path` 读 `min/max` 或 `q01/q99`。
|
| 103 |
+
2. 对 `actions` 归一化到 `[-1,1]`。
|
| 104 |
+
3. 保留 `raw_actions`,给评估和反归一化使用。
|
| 105 |
+
|
| 106 |
+
对应代码:
|
| 107 |
+
- `src/psi/config/transform.py:408-578`
|
| 108 |
+
|
| 109 |
+
### model(构造 VLM 输入)
|
| 110 |
+
|
| 111 |
+
`Psi0ModelTransform`:
|
| 112 |
+
|
| 113 |
+
1. 图像 resize/crop/aug。
|
| 114 |
+
2. 构造 chat message:`[images] + text instruction`。
|
| 115 |
+
3. `vlm_processor(...)` 得到 `input_ids`, `attention_mask`, `pixel_values`, `image_grid_thw`。
|
| 116 |
+
4. 附带 `actions/states/actions_mask/raw_actions` 回传。
|
| 117 |
+
|
| 118 |
+
对应代码:
|
| 119 |
+
- `src/psi/config/transform.py:797-888`
|
| 120 |
+
|
| 121 |
+
## 2.4 collator 到 batch
|
| 122 |
+
|
| 123 |
+
`PaddedCollatorForTogether` 负责:
|
| 124 |
+
|
| 125 |
+
1. 对 `input_ids` padding。
|
| 126 |
+
2. 组装 `attention_mask`。
|
| 127 |
+
3. 强制 stack `pixel_values` 和 `image_grid_thw`。
|
| 128 |
+
4. stack `actions/states/actions_mask`。
|
| 129 |
+
|
| 130 |
+
这意味着目前代码路径默认“有视觉输入字段”。
|
| 131 |
+
|
| 132 |
+
对应代码:
|
| 133 |
+
- `src/psi/trainers/qwen3vl_mixin.py:87-157`
|
| 134 |
+
|
| 135 |
+
## 2.5 模型 forward(posttrain)
|
| 136 |
+
|
| 137 |
+
`PosttrainTrainer.forward_and_loss` 里调用:
|
| 138 |
+
|
| 139 |
+
1. 从 GT 动作加噪(flow 或 ddpm)。
|
| 140 |
+
2. `Psi0Model.forward(...)` 输入:
|
| 141 |
+
- `input_ids`, `attention_mask`, `pixel_values`, `image_grid_thw`
|
| 142 |
+
- `action_samples`(噪声动作)
|
| 143 |
+
- `states`
|
| 144 |
+
- `timestep`
|
| 145 |
+
3. `Psi0Model` ��跑 VLM 拿最后一层 hidden states,再喂 action expert(扩散动作头)。
|
| 146 |
+
4. 输出 `action_pred`,与目标噪声/速度算 MSE。
|
| 147 |
+
|
| 148 |
+
对应代码:
|
| 149 |
+
- `src/psi/trainers/posttrain.py:525-584`
|
| 150 |
+
- `src/psi/models/psi0.py:1602-1639`
|
| 151 |
+
|
| 152 |
+
## 2.6 训练损失与评估
|
| 153 |
+
|
| 154 |
+
1. loss 是动作维度 MSE,加 `actions_mask` 后按维度加权汇总。
|
| 155 |
+
2. `action_dim != 7` 时,默认均匀权重(你的 36 维会走这个分支)。
|
| 156 |
+
3. 评估里目前有固定分组切分 `[7,14,21]`,这是历史结构假设,对你的 36 维语义不一定成立。
|
| 157 |
+
|
| 158 |
+
对应代码:
|
| 159 |
+
- `src/psi/trainers/posttrain.py:55-64`
|
| 160 |
+
- `src/psi/trainers/posttrain.py:505-517`
|
| 161 |
+
|
| 162 |
+
---
|
| 163 |
+
|
| 164 |
+
## 3) 输入输出维度分别在哪里定义
|
| 165 |
+
|
| 166 |
+
### 3.1 动作/状态主维度
|
| 167 |
+
|
| 168 |
+
1. `--model.action-dim`:动作维度(你要设 36)。
|
| 169 |
+
2. `--model.odim`:状态维度(要和最终 `states` 最后一维一致)。
|
| 170 |
+
3. `--model.action-chunk-size`:每次预测 horizon(`Tp`)。
|
| 171 |
+
4. `--data.transform.repack.pad-action-dim`:repack 后动作 pad 到多少维。
|
| 172 |
+
5. `--data.transform.repack.pad-state-dim`:repack 后状态 pad 到多少维。
|
| 173 |
+
6. `--data.transform.field.pad-action-dim`:归一化统计也按这个维度对齐。
|
| 174 |
+
7. `--data.transform.field.pad-state-dim`:同上。
|
| 175 |
+
|
| 176 |
+
参考:
|
| 177 |
+
- `scripts/train/psi0/posttrain-he-psi0.sh:55-75`
|
| 178 |
+
- `src/psi/config/model_psi0.py:17-37`
|
| 179 |
+
- `src/psi/models/psi0.py:592-594, 757-761, 788, 1499-1503`
|
| 180 |
+
|
| 181 |
+
### 3.2 VLM hidden size 与 view_feature_dim
|
| 182 |
+
|
| 183 |
+
1. `ObservationProjection.views_proj = Linear(view_feature_dim -> hidden_dim)`。
|
| 184 |
+
2. 这里的 `view_feature_dim` 必须等于 VLM 最后一层 hidden size。
|
| 185 |
+
3. 当前脚本用 `--model.view_feature_dim=2048`(对应现有 Qwen3-VL-2B 配置)。
|
| 186 |
+
|
| 187 |
+
参考:
|
| 188 |
+
- `src/psi/models/psi0.py:499`
|
| 189 |
+
- `src/psi/models/psi0.py:1624, 1632`
|
| 190 |
+
- `scripts/train/psi0/posttrain-he-psi0.sh:75`
|
| 191 |
+
|
| 192 |
+
### 3.3 典型 batch 形状(当前 posttrain he 脚本)
|
| 193 |
+
|
| 194 |
+
1. `actions`: `(B, 16, 36)`
|
| 195 |
+
2. `states`: `(B, 1, 36)`(`observation_horizon=1`)
|
| 196 |
+
3. `input_ids`: `(B, L)`
|
| 197 |
+
4. `attention_mask`: `(B, L)`
|
| 198 |
+
5. `pixel_values`, `image_grid_thw`: 由 processor 决定
|
| 199 |
+
|
| 200 |
+
---
|
| 201 |
+
|
| 202 |
+
## 4) 面向你目标的最小改造方案
|
| 203 |
+
|
| 204 |
+
下面给两条路径,我建议先 A 再 B。
|
| 205 |
+
|
| 206 |
+
## 路径 A(最小改动,优先跑通)
|
| 207 |
+
|
| 208 |
+
核心思想:先保留现有视觉字段接口,不做“真正 text-only 张量路径”;数据侧喂一个固定占位图(dummy image),让模型主要学 text->motion。这样能最大复用 `Psi0ModelTransform + Collator + Psi0Model.forward`,代码改动最小。
|
| 209 |
+
|
| 210 |
+
你需要做:
|
| 211 |
+
|
| 212 |
+
1. 新增 `BoneSeedDataset`(或在现有 dataset 适配)输出统一字段:
|
| 213 |
+
- `observations`: 放一个固定 PIL 图(例如纯黑 240x320)
|
| 214 |
+
- `states`: `(To, 36)`,通常 `To=1`
|
| 215 |
+
- `actions`: `(Tp, 36)`
|
| 216 |
+
- `instruction`: caption
|
| 217 |
+
- `actions_mask`: `(Tp, 36)`
|
| 218 |
+
|
| 219 |
+
2. 新增一个 `DataConfig`(例如 `data_boneseed.py`)和一个训练 config(例如 `posttrain_boneseed_psi0_config.py`):
|
| 220 |
+
- `transform.repack`: 你的 `BoneSeedRepackTransform`
|
| 221 |
+
- `transform.field`: 复用 `ActionStateTransform`
|
| 222 |
+
- `transform.model`: 先复用 `Psi0ModelTransform`
|
| 223 |
+
|
| 224 |
+
3. 新增训练脚本(仿 `posttrain-he-psi0.sh`):
|
| 225 |
+
- 维度全设 36
|
| 226 |
+
- 先设置 `n_conditions=0`
|
| 227 |
+
- `tune_vlm=False`
|
| 228 |
+
- `noise_scheduler=flow`
|
| 229 |
+
|
| 230 |
+
优点:
|
| 231 |
+
|
| 232 |
+
1. 几乎不动模型与 trainer 主干。
|
| 233 |
+
2. 能最快 smoke test。
|
| 234 |
+
|
| 235 |
+
代价:
|
| 236 |
+
|
| 237 |
+
1. 严格来说不是“纯 text-only”实现,只是视觉分支恒定输入。
|
| 238 |
+
|
| 239 |
+
## 路径 B(真正 text-only)
|
| 240 |
+
|
| 241 |
+
需要改 3 个点:
|
| 242 |
+
|
| 243 |
+
1. `Psi0ModelTransform.build_qwenvl_inputs`:
|
| 244 |
+
- 允许 `imgs=[]`。
|
| 245 |
+
- `process_vision_info` 返回 `None` 时,仍可仅返回 text tokens。
|
| 246 |
+
|
| 247 |
+
2. `PaddedCollatorForTogether`:
|
| 248 |
+
- `pixel_values/image_grid_thw` 改成可选字段(存在才 stack)。
|
| 249 |
+
|
| 250 |
+
3. `Psi0Model.forward`:
|
| 251 |
+
- `pixel_values=None, image_grid_thw=None` 时也能调用 VLM。
|
| 252 |
+
- 对应 forward 参数改成 optional。
|
| 253 |
+
|
| 254 |
+
这是更“干净”的 text-to-motion 路径,但改动比 A 稍大。
|
| 255 |
+
|
| 256 |
+
---
|
| 257 |
+
|
| 258 |
+
## 5) Qwen-2.5-Omni-3B 替换要点
|
| 259 |
+
|
| 260 |
+
当前 posttrain 写死了 Qwen3 类:
|
| 261 |
+
|
| 262 |
+
1. `src/psi/trainers/posttrain.py` 里 import `Qwen3VLForConditionalGeneration`。
|
| 263 |
+
2. `src/psi/models/psi0.py` 类型标注也是 `Qwen3VLForConditionalGeneration`。
|
| 264 |
+
|
| 265 |
+
要换 Omni,最小做法:
|
| 266 |
+
|
| 267 |
+
1. 在 posttrain 里改成可切换 VLM 构造:
|
| 268 |
+
- `Qwen2_5OmniForConditionalGeneration` + `Qwen2_5OmniProcessor`(或 Auto 类统一封装)。
|
| 269 |
+
|
| 270 |
+
2. 保持 `output_hidden_states=True`,继续取最后一层 hidden states 作为 action expert 条件。
|
| 271 |
+
|
| 272 |
+
3. 校对 hidden size,并同步 `--model.view_feature_dim`。
|
| 273 |
+
|
| 274 |
+
你当前环境可用性(已验证):
|
| 275 |
+
|
| 276 |
+
1. `.venv-psi` 下 `transformers==4.57.0`。
|
| 277 |
+
2. 可 import:
|
| 278 |
+
- `Qwen2_5OmniForConditionalGeneration`
|
| 279 |
+
- `Qwen2_5OmniProcessor`
|
| 280 |
+
|
| 281 |
+
建议你自己再做一次快速确认:
|
| 282 |
+
|
| 283 |
+
```bash
|
| 284 |
+
.venv-psi/bin/python - <<'PY'
|
| 285 |
+
from transformers import Qwen2_5OmniForConditionalGeneration, Qwen2_5OmniProcessor
|
| 286 |
+
print("ok")
|
| 287 |
+
PY
|
| 288 |
+
```
|
| 289 |
+
|
| 290 |
+
再确认 hidden size(用于 `view_feature_dim`):
|
| 291 |
+
|
| 292 |
+
```bash
|
| 293 |
+
.venv-psi/bin/python - <<'PY'
|
| 294 |
+
from transformers import AutoConfig
|
| 295 |
+
cfg = AutoConfig.from_pretrained("Qwen/Qwen2.5-Omni-3B")
|
| 296 |
+
print("thinker hidden:", getattr(getattr(cfg, "thinker_config", None), "hidden_size", None))
|
| 297 |
+
print("hidden:", getattr(cfg, "hidden_size", None))
|
| 298 |
+
PY
|
| 299 |
+
```
|
| 300 |
+
|
| 301 |
+
---
|
| 302 |
+
|
| 303 |
+
## 6) 从数据处理开始的落地步骤(建议执行顺序)
|
| 304 |
+
|
| 305 |
+
1. 先定动作语义:
|
| 306 |
+
- 你的 36 维是绝对量还是 delta。
|
| 307 |
+
- 如果你数据已经是重定向后的绝对关节+位姿,优先 `use_delta_actions=False`,先简化问题。
|
| 308 |
+
|
| 309 |
+
2. 做 dataset 适配(最关键):
|
| 310 |
+
- 先写 `BoneSeedDataset.__getitem__`,把每个样本变成统一结构。
|
| 311 |
+
- 确保 `states/actions` 维度稳定。
|
| 312 |
+
- caption 映射到 `instruction`。
|
| 313 |
+
|
| 314 |
+
3. 生成统计文件(归一化必需):
|
| 315 |
+
- `ActionStateTransform` 需要 `stat_path`。
|
| 316 |
+
- 至少包含 `action` 键下的 `min/max` 或 `q01/q99`,维度 36。
|
| 317 |
+
|
| 318 |
+
4. 新建专用 config 与训练脚本:
|
| 319 |
+
- 先复用 posttrain trainer,不要改训练循环。
|
| 320 |
+
- 参数先直接对齐:`action_dim=36`, `odim=36`, `pad_*_dim=36`。
|
| 321 |
+
|
| 322 |
+
5. 再换 VLM 到 Omni:
|
| 323 |
+
- 先只做“文本+占位图”跑通。
|
| 324 |
+
- 然后再做严格 text-only 可选字段改造。
|
| 325 |
+
|
| 326 |
+
6. 最后做 smoke test:
|
| 327 |
+
- 1-2 个 batch overfit(`overfit_single_batch=True`)先看 loss 能不能快速下降。
|
| 328 |
+
|
| 329 |
+
---
|
| 330 |
+
|
| 331 |
+
## 7) 你现在就要注意的风险点
|
| 332 |
+
|
| 333 |
+
1. `HEPosttrainRepackTransform` 里有明显 typo:
|
| 334 |
+
- `obs_mask": datpa["obs_mask"]`,`datpa` 未定义。
|
| 335 |
+
- 位置:`src/psi/config/transform.py:766`
|
| 336 |
+
- 如果你复用这段逻辑,先修掉。
|
| 337 |
+
|
| 338 |
+
2. 评估指标分组写死(对 36 维不语义一致):
|
| 339 |
+
- `np.split(..., [7,14,21])`
|
| 340 |
+
- 位置:`src/psi/trainers/posttrain.py:512-517`
|
| 341 |
+
- 训练不一定受影响,但日志会误导。
|
| 342 |
+
|
| 343 |
+
3. 当前 collator/forward 对视觉字段是强依赖:
|
| 344 |
+
- 不改的话,text-only 会在 batch 组装处报错。
|
| 345 |
+
- 位置:`src/psi/trainers/qwen3vl_mixin.py:115-125`, `src/psi/models/psi0.py:1604-1622`
|
| 346 |
+
|
| 347 |
+
4. `view_feature_dim` 和 VLM hidden size 必须一致:
|
| 348 |
+
- 不一致会在线性层处 shape mismatch。
|
| 349 |
+
- 位置:`src/psi/models/psi0.py:499`
|
| 350 |
+
|
| 351 |
+
---
|
| 352 |
+
|
| 353 |
+
## 8) 最小改造文件清单(建议)
|
| 354 |
+
|
| 355 |
+
建议新增(而不是改老逻辑):
|
| 356 |
+
|
| 357 |
+
1. `src/psi/data/humanoid/boneseed_dataset.py`
|
| 358 |
+
2. `src/psi/config/data_boneseed.py`
|
| 359 |
+
3. `src/psi/config/train/posttrain_boneseed_psi0_config.py`
|
| 360 |
+
4. `scripts/train/psi0/posttrain-boneseed-psi0.sh`
|
| 361 |
+
|
| 362 |
+
如要做严格 text-only,再改:
|
| 363 |
+
|
| 364 |
+
1. `src/psi/config/transform.py`(`Psi0ModelTransform`)
|
| 365 |
+
2. `src/psi/trainers/qwen3vl_mixin.py`(`PaddedCollatorForTogether`)
|
| 366 |
+
3. `src/psi/models/psi0.py`(`Psi0Model.forward` optional vision args)
|
| 367 |
+
4. `src/psi/trainers/posttrain.py`(VLM 类替换成 Omni)
|
| 368 |
+
|
| 369 |
+
---
|
| 370 |
+
|
| 371 |
+
## 9) 一句话执行策略
|
| 372 |
+
|
| 373 |
+
先用“占位图 + 冻结 VLM + 只训 action expert”把 36 维 caption->motion 流程跑通,再把视觉输入改成 truly optional(text-only),最后替换到 Qwen2.5-Omni 并校准 `view_feature_dim`。
|
| 374 |
+
|
posttrain-v1-psi0-all-7task-tune-vlm-direct-dense-actionhead-L.sh
ADDED
|
@@ -0,0 +1,313 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
pkill -f monitor.py
|
| 2 |
+
|
| 3 |
+
cd /cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/Psi0-yth-v21
|
| 4 |
+
export PYTHONPATH=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/Psi0-yth-v21/src
|
| 5 |
+
export CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7
|
| 6 |
+
export PSI_OUTPUT_DIR=${PSI_OUTPUT_DIR:-/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/Psi0-yth-v21/runs}
|
| 7 |
+
export PSI_LOG_REPORT_TO=wandb
|
| 8 |
+
|
| 9 |
+
source /cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/Psi0-yth-v15/.venv-psi/bin/activate
|
| 10 |
+
[ -f .env ] && source .env
|
| 11 |
+
|
| 12 |
+
export WANDB_API_KEY="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"
|
| 13 |
+
|
| 14 |
+
export OMP_NUM_THREADS=8
|
| 15 |
+
export TF_CPP_MIN_LOG_LEVEL=3
|
| 16 |
+
|
| 17 |
+
# Keep JSON sources readable in this script, but pass compact JSON to tyro.
|
| 18 |
+
compact_json() {
|
| 19 |
+
python -c 'import json, sys; print(json.dumps(json.load(sys.stdin), separators=(",", ":")))'
|
| 20 |
+
}
|
| 21 |
+
|
| 22 |
+
PSI_SEED_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/Psi0/dataset/seed_all
|
| 23 |
+
PSI_SEED_METADATA=metadata/seed_metadata_v004.csv
|
| 24 |
+
|
| 25 |
+
PSI_DANCE_MOTION_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/GR00T-WholeBodyControl/gear_sonic_deploy/reference/converted/psi0_dance/gvhmr
|
| 26 |
+
PSI_DANCE_AUDIO_VIDEO_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/dance_1723_video_psi0/dance_1723_video_10s
|
| 27 |
+
PSI_DANCE_VIDEO_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/dance_1723_video_psi0/dance_1723_video_10s
|
| 28 |
+
PSI_DANCE_VIDEO_EMBED_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/dance_1723_video_psi0/dance_1723_video_10s_embeds
|
| 29 |
+
PSI_VIDEO_DEMO_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/videogen/motion_example_0506_demo
|
| 30 |
+
PSI_QWEN3_TTS_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/qwen3-tts
|
| 31 |
+
PSI_VIDEO_CLIP_DATASET_ROOT=${PSI_VIDEO_CLIP_DATASET_ROOT:-/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/videogen/psi0_video_clip_50f_dense_stride1_direct_full}
|
| 32 |
+
PSI_DANCE_VIDEO_CLIP_ROOT=$PSI_VIDEO_CLIP_DATASET_ROOT/dance_1723_video_10s
|
| 33 |
+
PSI_MOTION_EXAMPLE_VIDEO_CLIP_ROOT=$PSI_VIDEO_CLIP_DATASET_ROOT/motion_example_0506_demo
|
| 34 |
+
PSI_DANCE_VIDEO_CLIP_EMBED_ROOT=${PSI_DANCE_VIDEO_CLIP_EMBED_ROOT:-$PSI_DANCE_VIDEO_CLIP_ROOT/embeds}
|
| 35 |
+
PSI_MOTION_EXAMPLE_VIDEO_CLIP_EMBED_ROOT=${PSI_MOTION_EXAMPLE_VIDEO_CLIP_EMBED_ROOT:-$PSI_MOTION_EXAMPLE_VIDEO_CLIP_ROOT/embeds}
|
| 36 |
+
PSI_MOTION_EXAMPLE_VIDEO_EMBED_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/videogen/motion_example_0506_demo_video_embeds
|
| 37 |
+
|
| 38 |
+
if [ -z "$PSI_AUDIO_SOURCES_JSON" ]; then
|
| 39 |
+
PSI_AUDIO_SOURCES_JSON=$(compact_json <<JSON
|
| 40 |
+
[
|
| 41 |
+
{
|
| 42 |
+
"name": "psi0_dance_audio",
|
| 43 |
+
"root_dir": "$PSI_DANCE_MOTION_ROOT",
|
| 44 |
+
"video_dir": "$PSI_DANCE_AUDIO_VIDEO_ROOT",
|
| 45 |
+
"dataset_name": "seed-audio-g1"
|
| 46 |
+
}
|
| 47 |
+
]
|
| 48 |
+
JSON
|
| 49 |
+
)
|
| 50 |
+
fi
|
| 51 |
+
|
| 52 |
+
if [ -z "$PSI_AUDIO_COMMAND_SOURCES_JSON" ]; then
|
| 53 |
+
PSI_AUDIO_COMMAND_SOURCES_JSON=$(compact_json <<JSON
|
| 54 |
+
[
|
| 55 |
+
{
|
| 56 |
+
"name": "qwen3_tts_seed_audio_command",
|
| 57 |
+
"root_dir": "$PSI_SEED_ROOT",
|
| 58 |
+
"dataset_root": "$PSI_QWEN3_TTS_ROOT",
|
| 59 |
+
"metadata_jsonl": "metadata.jsonl",
|
| 60 |
+
"motion_dir_key": "filename",
|
| 61 |
+
"audio_path_key": "audio_path",
|
| 62 |
+
"instruction_key": "content_short_description",
|
| 63 |
+
"motion_metadata_csv": "$PSI_SEED_METADATA",
|
| 64 |
+
"motion_metadata_key": "filename",
|
| 65 |
+
"motion_metadata_path_key": "move_g1_path",
|
| 66 |
+
"audio_file_extensions": ".wav",
|
| 67 |
+
"audio_task": "command",
|
| 68 |
+
"dataset_name": "seed-audio-command-g1"
|
| 69 |
+
}
|
| 70 |
+
]
|
| 71 |
+
JSON
|
| 72 |
+
)
|
| 73 |
+
fi
|
| 74 |
+
|
| 75 |
+
if [ -z "$PSI_VIDEO_SOURCES_JSON" ]; then
|
| 76 |
+
PSI_VIDEO_SOURCES_JSON=$(compact_json <<JSON
|
| 77 |
+
[
|
| 78 |
+
{
|
| 79 |
+
"name": "psi0_dance_gvhmr",
|
| 80 |
+
"root_dir": "$PSI_DANCE_MOTION_ROOT",
|
| 81 |
+
"video_dir": "$PSI_DANCE_VIDEO_ROOT",
|
| 82 |
+
"video_embed_dir": "$PSI_DANCE_VIDEO_EMBED_ROOT",
|
| 83 |
+
"dataset_name": "seed-video-g1"
|
| 84 |
+
},
|
| 85 |
+
{
|
| 86 |
+
"name": "motion_example_0506_demo",
|
| 87 |
+
"dataset_root": "$PSI_VIDEO_DEMO_ROOT",
|
| 88 |
+
"metadata_jsonl": "metadata.jsonl",
|
| 89 |
+
"motion_dir_key": "converted_path",
|
| 90 |
+
"video_path_key": "video_path",
|
| 91 |
+
"instruction_key": "video_caption",
|
| 92 |
+
"video_embed_dir": "$PSI_MOTION_EXAMPLE_VIDEO_EMBED_ROOT",
|
| 93 |
+
"dataset_name": "motion-example-video-g1"
|
| 94 |
+
}
|
| 95 |
+
]
|
| 96 |
+
JSON
|
| 97 |
+
)
|
| 98 |
+
fi
|
| 99 |
+
|
| 100 |
+
if [ -z "$PSI_VIDEO_CLIP_SOURCES_JSON" ]; then
|
| 101 |
+
PSI_VIDEO_CLIP_SOURCES_JSON=$(compact_json <<JSON
|
| 102 |
+
[
|
| 103 |
+
{
|
| 104 |
+
"name": "dance_1723_video_10s_clip50",
|
| 105 |
+
"dataset_root": "$PSI_DANCE_VIDEO_CLIP_ROOT",
|
| 106 |
+
"metadata_jsonl": "metadata.jsonl",
|
| 107 |
+
"motion_dir_key": "converted_path",
|
| 108 |
+
"video_path_key": "video_path",
|
| 109 |
+
"instruction_key": "video_caption",
|
| 110 |
+
"video_embed_dir": "$PSI_DANCE_VIDEO_CLIP_EMBED_ROOT",
|
| 111 |
+
"dataset_name": "dance-video-clip-g1"
|
| 112 |
+
},
|
| 113 |
+
{
|
| 114 |
+
"name": "motion_example_0506_demo_clip50",
|
| 115 |
+
"dataset_root": "$PSI_MOTION_EXAMPLE_VIDEO_CLIP_ROOT",
|
| 116 |
+
"metadata_jsonl": "metadata.jsonl",
|
| 117 |
+
"motion_dir_key": "converted_path",
|
| 118 |
+
"video_path_key": "video_path",
|
| 119 |
+
"instruction_key": "video_caption",
|
| 120 |
+
"video_embed_dir": "$PSI_MOTION_EXAMPLE_VIDEO_CLIP_EMBED_ROOT",
|
| 121 |
+
"dataset_name": "motion-example-video-clip-g1"
|
| 122 |
+
}
|
| 123 |
+
]
|
| 124 |
+
JSON
|
| 125 |
+
)
|
| 126 |
+
fi
|
| 127 |
+
|
| 128 |
+
# 7-task from-scratch mix, excluding legacy VIDEO_TEXT / VIDEO_ONLY.
|
| 129 |
+
# Smaller data sources get larger weights. These use a tempered inverse-size
|
| 130 |
+
# rule, (47M / source_size)^0.25, so rare modalities are upweighted without
|
| 131 |
+
# completely starving text/video_clip in BatchMixtureSampler.
|
| 132 |
+
# text stride1 frame samples: ~47,000,000
|
| 133 |
+
# music audio dance frame samples: 213,731
|
| 134 |
+
# audio_command wav files: 41,760
|
| 135 |
+
# dense video_clip metadata: 4,597,576
|
| 136 |
+
PSI_TEXT_TASK_WEIGHT=1.0
|
| 137 |
+
PSI_AUDIO_TEXT_TASK_WEIGHT=3.8509
|
| 138 |
+
PSI_AUDIO_ONLY_TASK_WEIGHT=3.8509
|
| 139 |
+
PSI_AUDIO_COMMAND_TEXT_TASK_WEIGHT=5.7921
|
| 140 |
+
PSI_AUDIO_COMMAND_ONLY_TASK_WEIGHT=5.7921
|
| 141 |
+
PSI_VIDEO_TEXT_TASK_WEIGHT=0
|
| 142 |
+
PSI_VIDEO_ONLY_TASK_WEIGHT=0
|
| 143 |
+
PSI_VIDEO_CLIP_TEXT_TASK_WEIGHT=1.7881
|
| 144 |
+
PSI_VIDEO_CLIP_ONLY_TASK_WEIGHT=1.7881
|
| 145 |
+
|
| 146 |
+
PSI_PRETRAIN_BACKBONE_PATH=/cpfs01/VideoGen/public_ckpt/Qwen/Qwen2.5-Omni-3B
|
| 147 |
+
NUM_PAST_FRAMES=0
|
| 148 |
+
STATE_HORIZON=$((NUM_PAST_FRAMES + 1))
|
| 149 |
+
ACTION_CHUNK_SIZE=30
|
| 150 |
+
ACTION_EXEC_HORIZON=20
|
| 151 |
+
PROGRESS_DIM=5
|
| 152 |
+
ACTION_HIDDEN_DIM=1920
|
| 153 |
+
ACTION_NUM_BLOCKS=28
|
| 154 |
+
ACTION_NHEADS=20
|
| 155 |
+
ACTION_HEAD_DIM=$((ACTION_HIDDEN_DIM / ACTION_NHEADS))
|
| 156 |
+
VIDEO_SAMPLE_FPS=1.0
|
| 157 |
+
VIDEO_CLIP_SAMPLE_FPS=8.0
|
| 158 |
+
VIDEO_SIZE_PIXELS=50176
|
| 159 |
+
VIDEO_DECODER=ffmpeg
|
| 160 |
+
VIDEO_CACHE_SIZE=64
|
| 161 |
+
VIDEO_CLIP_MOTION_CACHE_SIZE=256
|
| 162 |
+
VIDEO_CLIP_TRUST_METADATA=${VIDEO_CLIP_TRUST_METADATA:-1}
|
| 163 |
+
VIDEO_CLIP_RECORDS_CACHE_DIR=${VIDEO_CLIP_RECORDS_CACHE_DIR:-$PSI_VIDEO_CLIP_DATASET_ROOT/.index_cache}
|
| 164 |
+
if [ "$VIDEO_CLIP_TRUST_METADATA" = "1" ] || [ "$VIDEO_CLIP_TRUST_METADATA" = "true" ] || [ "$VIDEO_CLIP_TRUST_METADATA" = "True" ]; then
|
| 165 |
+
VIDEO_CLIP_TRUST_METADATA_FLAG="--data.video-clip.trust-metadata"
|
| 166 |
+
else
|
| 167 |
+
VIDEO_CLIP_TRUST_METADATA_FLAG="--data.video-clip.no-trust-metadata"
|
| 168 |
+
fi
|
| 169 |
+
DATALOADER_NUM_WORKERS=4
|
| 170 |
+
VAL_DATALOADER_NUM_WORKERS=2
|
| 171 |
+
|
| 172 |
+
NPROC_PER_NODE=$(echo $CUDA_VISIBLE_DEVICES | tr ',' '\n' | wc -l)
|
| 173 |
+
|
| 174 |
+
echo "Training with $NPROC_PER_NODE GPU(s): $CUDA_VISIBLE_DEVICES"
|
| 175 |
+
echo "Text root: $PSI_SEED_ROOT"
|
| 176 |
+
echo "Dance motion root: $PSI_DANCE_MOTION_ROOT"
|
| 177 |
+
echo "Dance audio/video root: $PSI_DANCE_AUDIO_VIDEO_ROOT"
|
| 178 |
+
echo "Dance video root: $PSI_DANCE_VIDEO_ROOT"
|
| 179 |
+
echo "Dance video embed root: $PSI_DANCE_VIDEO_EMBED_ROOT"
|
| 180 |
+
echo "Qwen3-TTS acdudio command root: $PSI_QWEN3_TTS_ROOT"
|
| 181 |
+
echo "Audio sources: $PSI_AUDIO_SOURCES_JSON"
|
| 182 |
+
echo "Audio command sources: $PSI_AUDIO_COMMAND_SOURCES_JSON"
|
| 183 |
+
echo "Video sources: $PSI_VIDEO_SOURCES_JSON"
|
| 184 |
+
echo "Video clip sources: $PSI_VIDEO_CLIP_SOURCES_JSON"
|
| 185 |
+
echo "Task weights: text=$PSI_TEXT_TASK_WEIGHT audio_text=$PSI_AUDIO_TEXT_TASK_WEIGHT audio_only=$PSI_AUDIO_ONLY_TASK_WEIGHT audio_command_text=$PSI_AUDIO_COMMAND_TEXT_TASK_WEIGHT audio_command_only=$PSI_AUDIO_COMMAND_ONLY_TASK_WEIGHT video_text=$PSI_VIDEO_TEXT_TASK_WEIGHT video_only=$PSI_VIDEO_ONLY_TASK_WEIGHT video_clip_text=$PSI_VIDEO_CLIP_TEXT_TASK_WEIGHT video_clip_only=$PSI_VIDEO_CLIP_ONLY_TASK_WEIGHT"
|
| 186 |
+
# echo "Resume run dir: $PSI_PRETRAIN_RUN_DIR"
|
| 187 |
+
echo "TextOp 39D action/state enabled"
|
| 188 |
+
echo "Future progress embedding: enabled, dim $PROGRESS_DIM"
|
| 189 |
+
echo "Action head scale: L hidden=$ACTION_HIDDEN_DIM depth=$ACTION_NUM_BLOCKS heads=$ACTION_NHEADS head_dim=$ACTION_HEAD_DIM"
|
| 190 |
+
echo "Backbone path: $PSI_PRETRAIN_BACKBONE_PATH"
|
| 191 |
+
echo "Output dir: $PSI_OUTPUT_DIR"
|
| 192 |
+
echo "Video clip trust metadata: $VIDEO_CLIP_TRUST_METADATA"
|
| 193 |
+
echo "Video clip records cache dir: $VIDEO_CLIP_RECORDS_CACHE_DIR"
|
| 194 |
+
|
| 195 |
+
args="posttrain_v1_seed_text_audio_video_mix_psi0_config \
|
| 196 |
+
--seed=292285 \
|
| 197 |
+
--exp=posttrain_actionhead_L \
|
| 198 |
+
--timestamp=$(date +"%y%m%d%H%M") \
|
| 199 |
+
--train.name=posttrain_v1_trimodal \
|
| 200 |
+
--train.data_parallel=ddp \
|
| 201 |
+
--train.mixed_precision=bf16 \
|
| 202 |
+
--train.train_batch_size=16 \
|
| 203 |
+
--train.val_batch_size=32 \
|
| 204 |
+
--train.num-workers=$DATALOADER_NUM_WORKERS \
|
| 205 |
+
--train.val-num-workers=$VAL_DATALOADER_NUM_WORKERS \
|
| 206 |
+
--train.resume_from_checkpoint=None \
|
| 207 |
+
--train.max_checkpoints_to_keep=1 \
|
| 208 |
+
--train.gradient_accumulation_steps=1 \
|
| 209 |
+
--train.learning_rate=1e-4 \
|
| 210 |
+
--train.max_training_steps=500000 \
|
| 211 |
+
--train.warmup_ratio=None \
|
| 212 |
+
--train.warmup_steps=1000 \
|
| 213 |
+
--train.checkpointing_steps=1000 \
|
| 214 |
+
--train.validation_steps=1000 \
|
| 215 |
+
--train.val_num_batches=20 \
|
| 216 |
+
--train.max_grad_norm=1.0 \
|
| 217 |
+
--train.lr_scheduler_type=cosine_with_min_lr \
|
| 218 |
+
--train.lr_scheduler_kwargs.betas 0.9 0.999 \
|
| 219 |
+
--train.lr_scheduler_kwargs.weight_decay=0 \
|
| 220 |
+
--train.lr_scheduler_kwargs.eps=1e-8 \
|
| 221 |
+
--log.report_to=$PSI_LOG_REPORT_TO \
|
| 222 |
+
--train.output_dir=$PSI_OUTPUT_DIR \
|
| 223 |
+
--data.text.root-dir=$PSI_SEED_ROOT \
|
| 224 |
+
--data.text.metadata-csv=$PSI_SEED_METADATA \
|
| 225 |
+
--data.text-ratio=$PSI_TEXT_TASK_WEIGHT \
|
| 226 |
+
--data.audio.sources-json=$PSI_AUDIO_SOURCES_JSON \
|
| 227 |
+
--data.audio-ratio=$PSI_AUDIO_TEXT_TASK_WEIGHT \
|
| 228 |
+
--data.audio-only-ratio=$PSI_AUDIO_ONLY_TASK_WEIGHT \
|
| 229 |
+
--data.audio.sample-stride=1 \
|
| 230 |
+
--data.audio.motion-fps=50 \
|
| 231 |
+
--data.audio.audio-sampling-rate=16000 \
|
| 232 |
+
--data.audio.audio-window-seconds=1.0 \
|
| 233 |
+
--data.audio.audio-left-context-seconds=0.0 \
|
| 234 |
+
--data.audio.full-audio \
|
| 235 |
+
--data.audio-command.sources-json=$PSI_AUDIO_COMMAND_SOURCES_JSON \
|
| 236 |
+
--data.audio-command-ratio=$PSI_AUDIO_COMMAND_TEXT_TASK_WEIGHT \
|
| 237 |
+
--data.audio-command-only-ratio=$PSI_AUDIO_COMMAND_ONLY_TASK_WEIGHT \
|
| 238 |
+
--data.audio-command.sample-stride=1 \
|
| 239 |
+
--data.audio-command.motion-fps=50 \
|
| 240 |
+
--data.audio-command.audio-sampling-rate=16000 \
|
| 241 |
+
--data.audio-command.audio-window-seconds=1.0 \
|
| 242 |
+
--data.audio-command.audio-left-context-seconds=0.0 \
|
| 243 |
+
--data.audio-command.full-audio \
|
| 244 |
+
--data.video-ratio=$PSI_VIDEO_TEXT_TASK_WEIGHT \
|
| 245 |
+
--data.video-only-ratio=$PSI_VIDEO_ONLY_TASK_WEIGHT \
|
| 246 |
+
--data.video.sample-stride=1 \
|
| 247 |
+
--data.video.motion-fps=50 \
|
| 248 |
+
--data.video.video-sample-fps=$VIDEO_SAMPLE_FPS \
|
| 249 |
+
--data.video.video-size-pixels=$VIDEO_SIZE_PIXELS \
|
| 250 |
+
--data.video.video-decoder=$VIDEO_DECODER \
|
| 251 |
+
--data.video.video-cache-size=$VIDEO_CACHE_SIZE \
|
| 252 |
+
--data.video.sources-json=$PSI_VIDEO_SOURCES_JSON \
|
| 253 |
+
--data.video-clip-text-ratio=$PSI_VIDEO_CLIP_TEXT_TASK_WEIGHT \
|
| 254 |
+
--data.video-clip-only-ratio=$PSI_VIDEO_CLIP_ONLY_TASK_WEIGHT \
|
| 255 |
+
--data.video-clip.sources-json=$PSI_VIDEO_CLIP_SOURCES_JSON \
|
| 256 |
+
--data.video-clip.sample-stride=1 \
|
| 257 |
+
--data.video-clip.motion-fps=50 \
|
| 258 |
+
--data.video-clip.video-sample-fps=$VIDEO_CLIP_SAMPLE_FPS \
|
| 259 |
+
--data.video-clip.video-size-pixels=$VIDEO_SIZE_PIXELS \
|
| 260 |
+
--data.video-clip.video-decoder=$VIDEO_DECODER \
|
| 261 |
+
--data.video-clip.video-cache-size=$VIDEO_CACHE_SIZE \
|
| 262 |
+
--data.video-clip.motion-cache-size=$VIDEO_CLIP_MOTION_CACHE_SIZE \
|
| 263 |
+
$VIDEO_CLIP_TRUST_METADATA_FLAG \
|
| 264 |
+
--data.video-clip.records-cache-dir=$VIDEO_CLIP_RECORDS_CACHE_DIR \
|
| 265 |
+
--data.video-clip.action-exec-horizon=$ACTION_EXEC_HORIZON \
|
| 266 |
+
--data.sampler=batch_mixture \
|
| 267 |
+
--data.val-task=text,audio,audio_only,audio_command,audio_command_only,video_clip,video_clip_only \
|
| 268 |
+
--data.transform.repack.num-past-frames=$NUM_PAST_FRAMES \
|
| 269 |
+
--data.transform.repack.textop-actions \
|
| 270 |
+
--data.transform.repack.action-chunk-size=$ACTION_CHUNK_SIZE \
|
| 271 |
+
--data.transform.repack.pad-action-dim=39 \
|
| 272 |
+
--data.transform.repack.pad-state-dim=39 \
|
| 273 |
+
--data.transform.field.action_norm_type=bounds_q99 \
|
| 274 |
+
--data.transform.field.stat-path=assets/stats/seed_all_g1_motion39_textop_stats.json \
|
| 275 |
+
--data.transform.field.normalize-state \
|
| 276 |
+
--data.transform.field.pad-action-dim=39 \
|
| 277 |
+
--data.transform.field.pad-state-dim=39 \
|
| 278 |
+
--data.transform.model.resize.size 240 320 \
|
| 279 |
+
--data.transform.model.center_crop.size 240 320 \
|
| 280 |
+
--data.transform.model.no-img-aug \
|
| 281 |
+
--data.transform.model.text-only \
|
| 282 |
+
--data.transform.model.audio-only \
|
| 283 |
+
--data.transform.model.video-only \
|
| 284 |
+
--data.transform.model.video-size-pixels=$VIDEO_SIZE_PIXELS \
|
| 285 |
+
--model.model_name_or_path=$PSI_PRETRAIN_BACKBONE_PATH \
|
| 286 |
+
--model.noise-scheduler=flow \
|
| 287 |
+
--model.n_conditions=0 \
|
| 288 |
+
--model.action-chunk-size=$ACTION_CHUNK_SIZE \
|
| 289 |
+
--model.action-dim=39 \
|
| 290 |
+
--model.action-exec-horizon=$ACTION_EXEC_HORIZON \
|
| 291 |
+
--model.observation-horizon=$STATE_HORIZON \
|
| 292 |
+
--model.odim=39 \
|
| 293 |
+
--model.view_feature_dim=2048 \
|
| 294 |
+
--model.hidden-dim=$ACTION_HIDDEN_DIM \
|
| 295 |
+
--model.num-blocks=$ACTION_NUM_BLOCKS \
|
| 296 |
+
--model.nhead=$ACTION_NHEADS \
|
| 297 |
+
--model.use-future-progress-embedding \
|
| 298 |
+
--model.future-progress-dim=$PROGRESS_DIM \
|
| 299 |
+
--model.tune-vlm \
|
| 300 |
+
--model.lang-backbone-lr=1e-5 \
|
| 301 |
+
--model.no-use_film \
|
| 302 |
+
--model.no-combined_temb \
|
| 303 |
+
--model.rtc \
|
| 304 |
+
--model.max-delay=11 \
|
| 305 |
+
--train.max-grad-value=1 \
|
| 306 |
+
--train.loss-group-weight.joint-abs=1.0 \
|
| 307 |
+
--train.loss-group-weight.root-delta-xy=3.0 \
|
| 308 |
+
--train.loss-group-weight.root-rot5-delta=10.0 \
|
| 309 |
+
--train.loss-group-weight.root-height=15.0 \
|
| 310 |
+
--train.loss-group-weight.foot-contact=0.1
|
| 311 |
+
"
|
| 312 |
+
torchrun --nproc_per_node=$NPROC_PER_NODE --master_port=29500 scripts/train.py \
|
| 313 |
+
$args "$@"
|
posttrain-v1-psi0-all-7task-tune-vlm-direct-dense-actionhead-M.sh
ADDED
|
@@ -0,0 +1,313 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
pkill -f monitor.py
|
| 2 |
+
|
| 3 |
+
cd /cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/Psi0-yth-v21
|
| 4 |
+
export PYTHONPATH=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/Psi0-yth-v21/src
|
| 5 |
+
export CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7
|
| 6 |
+
export PSI_OUTPUT_DIR=${PSI_OUTPUT_DIR:-/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/Psi0-yth-v21/runs}
|
| 7 |
+
export PSI_LOG_REPORT_TO=wandb
|
| 8 |
+
|
| 9 |
+
source /cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/Psi0-yth-v15/.venv-psi/bin/activate
|
| 10 |
+
[ -f .env ] && source .env
|
| 11 |
+
|
| 12 |
+
export WANDB_API_KEY="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"
|
| 13 |
+
|
| 14 |
+
export OMP_NUM_THREADS=8
|
| 15 |
+
export TF_CPP_MIN_LOG_LEVEL=3
|
| 16 |
+
|
| 17 |
+
# Keep JSON sources readable in this script, but pass compact JSON to tyro.
|
| 18 |
+
compact_json() {
|
| 19 |
+
python -c 'import json, sys; print(json.dumps(json.load(sys.stdin), separators=(",", ":")))'
|
| 20 |
+
}
|
| 21 |
+
|
| 22 |
+
PSI_SEED_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/Psi0/dataset/seed_all
|
| 23 |
+
PSI_SEED_METADATA=metadata/seed_metadata_v004.csv
|
| 24 |
+
|
| 25 |
+
PSI_DANCE_MOTION_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/GR00T-WholeBodyControl/gear_sonic_deploy/reference/converted/psi0_dance/gvhmr
|
| 26 |
+
PSI_DANCE_AUDIO_VIDEO_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/dance_1723_video_psi0/dance_1723_video_10s
|
| 27 |
+
PSI_DANCE_VIDEO_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/dance_1723_video_psi0/dance_1723_video_10s
|
| 28 |
+
PSI_DANCE_VIDEO_EMBED_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/dance_1723_video_psi0/dance_1723_video_10s_embeds
|
| 29 |
+
PSI_VIDEO_DEMO_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/videogen/motion_example_0506_demo
|
| 30 |
+
PSI_QWEN3_TTS_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/qwen3-tts
|
| 31 |
+
PSI_VIDEO_CLIP_DATASET_ROOT=${PSI_VIDEO_CLIP_DATASET_ROOT:-/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/videogen/psi0_video_clip_50f_dense_stride1_direct_full}
|
| 32 |
+
PSI_DANCE_VIDEO_CLIP_ROOT=$PSI_VIDEO_CLIP_DATASET_ROOT/dance_1723_video_10s
|
| 33 |
+
PSI_MOTION_EXAMPLE_VIDEO_CLIP_ROOT=$PSI_VIDEO_CLIP_DATASET_ROOT/motion_example_0506_demo
|
| 34 |
+
PSI_DANCE_VIDEO_CLIP_EMBED_ROOT=${PSI_DANCE_VIDEO_CLIP_EMBED_ROOT:-$PSI_DANCE_VIDEO_CLIP_ROOT/embeds}
|
| 35 |
+
PSI_MOTION_EXAMPLE_VIDEO_CLIP_EMBED_ROOT=${PSI_MOTION_EXAMPLE_VIDEO_CLIP_EMBED_ROOT:-$PSI_MOTION_EXAMPLE_VIDEO_CLIP_ROOT/embeds}
|
| 36 |
+
PSI_MOTION_EXAMPLE_VIDEO_EMBED_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/videogen/motion_example_0506_demo_video_embeds
|
| 37 |
+
|
| 38 |
+
if [ -z "$PSI_AUDIO_SOURCES_JSON" ]; then
|
| 39 |
+
PSI_AUDIO_SOURCES_JSON=$(compact_json <<JSON
|
| 40 |
+
[
|
| 41 |
+
{
|
| 42 |
+
"name": "psi0_dance_audio",
|
| 43 |
+
"root_dir": "$PSI_DANCE_MOTION_ROOT",
|
| 44 |
+
"video_dir": "$PSI_DANCE_AUDIO_VIDEO_ROOT",
|
| 45 |
+
"dataset_name": "seed-audio-g1"
|
| 46 |
+
}
|
| 47 |
+
]
|
| 48 |
+
JSON
|
| 49 |
+
)
|
| 50 |
+
fi
|
| 51 |
+
|
| 52 |
+
if [ -z "$PSI_AUDIO_COMMAND_SOURCES_JSON" ]; then
|
| 53 |
+
PSI_AUDIO_COMMAND_SOURCES_JSON=$(compact_json <<JSON
|
| 54 |
+
[
|
| 55 |
+
{
|
| 56 |
+
"name": "qwen3_tts_seed_audio_command",
|
| 57 |
+
"root_dir": "$PSI_SEED_ROOT",
|
| 58 |
+
"dataset_root": "$PSI_QWEN3_TTS_ROOT",
|
| 59 |
+
"metadata_jsonl": "metadata.jsonl",
|
| 60 |
+
"motion_dir_key": "filename",
|
| 61 |
+
"audio_path_key": "audio_path",
|
| 62 |
+
"instruction_key": "content_short_description",
|
| 63 |
+
"motion_metadata_csv": "$PSI_SEED_METADATA",
|
| 64 |
+
"motion_metadata_key": "filename",
|
| 65 |
+
"motion_metadata_path_key": "move_g1_path",
|
| 66 |
+
"audio_file_extensions": ".wav",
|
| 67 |
+
"audio_task": "command",
|
| 68 |
+
"dataset_name": "seed-audio-command-g1"
|
| 69 |
+
}
|
| 70 |
+
]
|
| 71 |
+
JSON
|
| 72 |
+
)
|
| 73 |
+
fi
|
| 74 |
+
|
| 75 |
+
if [ -z "$PSI_VIDEO_SOURCES_JSON" ]; then
|
| 76 |
+
PSI_VIDEO_SOURCES_JSON=$(compact_json <<JSON
|
| 77 |
+
[
|
| 78 |
+
{
|
| 79 |
+
"name": "psi0_dance_gvhmr",
|
| 80 |
+
"root_dir": "$PSI_DANCE_MOTION_ROOT",
|
| 81 |
+
"video_dir": "$PSI_DANCE_VIDEO_ROOT",
|
| 82 |
+
"video_embed_dir": "$PSI_DANCE_VIDEO_EMBED_ROOT",
|
| 83 |
+
"dataset_name": "seed-video-g1"
|
| 84 |
+
},
|
| 85 |
+
{
|
| 86 |
+
"name": "motion_example_0506_demo",
|
| 87 |
+
"dataset_root": "$PSI_VIDEO_DEMO_ROOT",
|
| 88 |
+
"metadata_jsonl": "metadata.jsonl",
|
| 89 |
+
"motion_dir_key": "converted_path",
|
| 90 |
+
"video_path_key": "video_path",
|
| 91 |
+
"instruction_key": "video_caption",
|
| 92 |
+
"video_embed_dir": "$PSI_MOTION_EXAMPLE_VIDEO_EMBED_ROOT",
|
| 93 |
+
"dataset_name": "motion-example-video-g1"
|
| 94 |
+
}
|
| 95 |
+
]
|
| 96 |
+
JSON
|
| 97 |
+
)
|
| 98 |
+
fi
|
| 99 |
+
|
| 100 |
+
if [ -z "$PSI_VIDEO_CLIP_SOURCES_JSON" ]; then
|
| 101 |
+
PSI_VIDEO_CLIP_SOURCES_JSON=$(compact_json <<JSON
|
| 102 |
+
[
|
| 103 |
+
{
|
| 104 |
+
"name": "dance_1723_video_10s_clip50",
|
| 105 |
+
"dataset_root": "$PSI_DANCE_VIDEO_CLIP_ROOT",
|
| 106 |
+
"metadata_jsonl": "metadata.jsonl",
|
| 107 |
+
"motion_dir_key": "converted_path",
|
| 108 |
+
"video_path_key": "video_path",
|
| 109 |
+
"instruction_key": "video_caption",
|
| 110 |
+
"video_embed_dir": "$PSI_DANCE_VIDEO_CLIP_EMBED_ROOT",
|
| 111 |
+
"dataset_name": "dance-video-clip-g1"
|
| 112 |
+
},
|
| 113 |
+
{
|
| 114 |
+
"name": "motion_example_0506_demo_clip50",
|
| 115 |
+
"dataset_root": "$PSI_MOTION_EXAMPLE_VIDEO_CLIP_ROOT",
|
| 116 |
+
"metadata_jsonl": "metadata.jsonl",
|
| 117 |
+
"motion_dir_key": "converted_path",
|
| 118 |
+
"video_path_key": "video_path",
|
| 119 |
+
"instruction_key": "video_caption",
|
| 120 |
+
"video_embed_dir": "$PSI_MOTION_EXAMPLE_VIDEO_CLIP_EMBED_ROOT",
|
| 121 |
+
"dataset_name": "motion-example-video-clip-g1"
|
| 122 |
+
}
|
| 123 |
+
]
|
| 124 |
+
JSON
|
| 125 |
+
)
|
| 126 |
+
fi
|
| 127 |
+
|
| 128 |
+
# 7-task from-scratch mix, excluding legacy VIDEO_TEXT / VIDEO_ONLY.
|
| 129 |
+
# Smaller data sources get larger weights. These use a tempered inverse-size
|
| 130 |
+
# rule, (47M / source_size)^0.25, so rare modalities are upweighted without
|
| 131 |
+
# completely starving text/video_clip in BatchMixtureSampler.
|
| 132 |
+
# text stride1 frame samples: ~47,000,000
|
| 133 |
+
# music audio dance frame samples: 213,731
|
| 134 |
+
# audio_command wav files: 41,760
|
| 135 |
+
# dense video_clip metadata: 4,597,576
|
| 136 |
+
PSI_TEXT_TASK_WEIGHT=1.0
|
| 137 |
+
PSI_AUDIO_TEXT_TASK_WEIGHT=3.8509
|
| 138 |
+
PSI_AUDIO_ONLY_TASK_WEIGHT=3.8509
|
| 139 |
+
PSI_AUDIO_COMMAND_TEXT_TASK_WEIGHT=5.7921
|
| 140 |
+
PSI_AUDIO_COMMAND_ONLY_TASK_WEIGHT=5.7921
|
| 141 |
+
PSI_VIDEO_TEXT_TASK_WEIGHT=0
|
| 142 |
+
PSI_VIDEO_ONLY_TASK_WEIGHT=0
|
| 143 |
+
PSI_VIDEO_CLIP_TEXT_TASK_WEIGHT=1.7881
|
| 144 |
+
PSI_VIDEO_CLIP_ONLY_TASK_WEIGHT=1.7881
|
| 145 |
+
|
| 146 |
+
PSI_PRETRAIN_BACKBONE_PATH=/cpfs01/VideoGen/public_ckpt/Qwen/Qwen2.5-Omni-3B
|
| 147 |
+
NUM_PAST_FRAMES=0
|
| 148 |
+
STATE_HORIZON=$((NUM_PAST_FRAMES + 1))
|
| 149 |
+
ACTION_CHUNK_SIZE=30
|
| 150 |
+
ACTION_EXEC_HORIZON=20
|
| 151 |
+
PROGRESS_DIM=5
|
| 152 |
+
ACTION_HIDDEN_DIM=1152
|
| 153 |
+
ACTION_NUM_BLOCKS=16
|
| 154 |
+
ACTION_NHEADS=12
|
| 155 |
+
ACTION_HEAD_DIM=$((ACTION_HIDDEN_DIM / ACTION_NHEADS))
|
| 156 |
+
VIDEO_SAMPLE_FPS=1.0
|
| 157 |
+
VIDEO_CLIP_SAMPLE_FPS=8.0
|
| 158 |
+
VIDEO_SIZE_PIXELS=50176
|
| 159 |
+
VIDEO_DECODER=ffmpeg
|
| 160 |
+
VIDEO_CACHE_SIZE=64
|
| 161 |
+
VIDEO_CLIP_MOTION_CACHE_SIZE=256
|
| 162 |
+
VIDEO_CLIP_TRUST_METADATA=${VIDEO_CLIP_TRUST_METADATA:-1}
|
| 163 |
+
VIDEO_CLIP_RECORDS_CACHE_DIR=${VIDEO_CLIP_RECORDS_CACHE_DIR:-$PSI_VIDEO_CLIP_DATASET_ROOT/.index_cache}
|
| 164 |
+
if [ "$VIDEO_CLIP_TRUST_METADATA" = "1" ] || [ "$VIDEO_CLIP_TRUST_METADATA" = "true" ] || [ "$VIDEO_CLIP_TRUST_METADATA" = "True" ]; then
|
| 165 |
+
VIDEO_CLIP_TRUST_METADATA_FLAG="--data.video-clip.trust-metadata"
|
| 166 |
+
else
|
| 167 |
+
VIDEO_CLIP_TRUST_METADATA_FLAG="--data.video-clip.no-trust-metadata"
|
| 168 |
+
fi
|
| 169 |
+
DATALOADER_NUM_WORKERS=4
|
| 170 |
+
VAL_DATALOADER_NUM_WORKERS=2
|
| 171 |
+
|
| 172 |
+
NPROC_PER_NODE=$(echo $CUDA_VISIBLE_DEVICES | tr ',' '\n' | wc -l)
|
| 173 |
+
|
| 174 |
+
echo "Training with $NPROC_PER_NODE GPU(s): $CUDA_VISIBLE_DEVICES"
|
| 175 |
+
echo "Text root: $PSI_SEED_ROOT"
|
| 176 |
+
echo "Dance motion root: $PSI_DANCE_MOTION_ROOT"
|
| 177 |
+
echo "Dance audio/video root: $PSI_DANCE_AUDIO_VIDEO_ROOT"
|
| 178 |
+
echo "Dance video root: $PSI_DANCE_VIDEO_ROOT"
|
| 179 |
+
echo "Dance video embed root: $PSI_DANCE_VIDEO_EMBED_ROOT"
|
| 180 |
+
echo "Qwen3-TTS acdudio command root: $PSI_QWEN3_TTS_ROOT"
|
| 181 |
+
echo "Audio sources: $PSI_AUDIO_SOURCES_JSON"
|
| 182 |
+
echo "Audio command sources: $PSI_AUDIO_COMMAND_SOURCES_JSON"
|
| 183 |
+
echo "Video sources: $PSI_VIDEO_SOURCES_JSON"
|
| 184 |
+
echo "Video clip sources: $PSI_VIDEO_CLIP_SOURCES_JSON"
|
| 185 |
+
echo "Task weights: text=$PSI_TEXT_TASK_WEIGHT audio_text=$PSI_AUDIO_TEXT_TASK_WEIGHT audio_only=$PSI_AUDIO_ONLY_TASK_WEIGHT audio_command_text=$PSI_AUDIO_COMMAND_TEXT_TASK_WEIGHT audio_command_only=$PSI_AUDIO_COMMAND_ONLY_TASK_WEIGHT video_text=$PSI_VIDEO_TEXT_TASK_WEIGHT video_only=$PSI_VIDEO_ONLY_TASK_WEIGHT video_clip_text=$PSI_VIDEO_CLIP_TEXT_TASK_WEIGHT video_clip_only=$PSI_VIDEO_CLIP_ONLY_TASK_WEIGHT"
|
| 186 |
+
# echo "Resume run dir: $PSI_PRETRAIN_RUN_DIR"
|
| 187 |
+
echo "TextOp 39D action/state enabled"
|
| 188 |
+
echo "Future progress embedding: enabled, dim $PROGRESS_DIM"
|
| 189 |
+
echo "Action head scale: M hidden=$ACTION_HIDDEN_DIM depth=$ACTION_NUM_BLOCKS heads=$ACTION_NHEADS head_dim=$ACTION_HEAD_DIM"
|
| 190 |
+
echo "Backbone path: $PSI_PRETRAIN_BACKBONE_PATH"
|
| 191 |
+
echo "Output dir: $PSI_OUTPUT_DIR"
|
| 192 |
+
echo "Video clip trust metadata: $VIDEO_CLIP_TRUST_METADATA"
|
| 193 |
+
echo "Video clip records cache dir: $VIDEO_CLIP_RECORDS_CACHE_DIR"
|
| 194 |
+
|
| 195 |
+
args="posttrain_v1_seed_text_audio_video_mix_psi0_config \
|
| 196 |
+
--seed=292285 \
|
| 197 |
+
--exp=posttrain_actionhead_M \
|
| 198 |
+
--timestamp=$(date +"%y%m%d%H%M") \
|
| 199 |
+
--train.name=posttrain_v1_trimodal \
|
| 200 |
+
--train.data_parallel=ddp \
|
| 201 |
+
--train.mixed_precision=bf16 \
|
| 202 |
+
--train.train_batch_size=32 \
|
| 203 |
+
--train.val_batch_size=64 \
|
| 204 |
+
--train.num-workers=$DATALOADER_NUM_WORKERS \
|
| 205 |
+
--train.val-num-workers=$VAL_DATALOADER_NUM_WORKERS \
|
| 206 |
+
--train.resume_from_checkpoint=None \
|
| 207 |
+
--train.max_checkpoints_to_keep=1 \
|
| 208 |
+
--train.gradient_accumulation_steps=1 \
|
| 209 |
+
--train.learning_rate=1e-4 \
|
| 210 |
+
--train.max_training_steps=500000 \
|
| 211 |
+
--train.warmup_ratio=None \
|
| 212 |
+
--train.warmup_steps=1000 \
|
| 213 |
+
--train.checkpointing_steps=1000 \
|
| 214 |
+
--train.validation_steps=1000 \
|
| 215 |
+
--train.val_num_batches=20 \
|
| 216 |
+
--train.max_grad_norm=1.0 \
|
| 217 |
+
--train.lr_scheduler_type=cosine_with_min_lr \
|
| 218 |
+
--train.lr_scheduler_kwargs.betas 0.9 0.999 \
|
| 219 |
+
--train.lr_scheduler_kwargs.weight_decay=0 \
|
| 220 |
+
--train.lr_scheduler_kwargs.eps=1e-8 \
|
| 221 |
+
--log.report_to=$PSI_LOG_REPORT_TO \
|
| 222 |
+
--train.output_dir=$PSI_OUTPUT_DIR \
|
| 223 |
+
--data.text.root-dir=$PSI_SEED_ROOT \
|
| 224 |
+
--data.text.metadata-csv=$PSI_SEED_METADATA \
|
| 225 |
+
--data.text-ratio=$PSI_TEXT_TASK_WEIGHT \
|
| 226 |
+
--data.audio.sources-json=$PSI_AUDIO_SOURCES_JSON \
|
| 227 |
+
--data.audio-ratio=$PSI_AUDIO_TEXT_TASK_WEIGHT \
|
| 228 |
+
--data.audio-only-ratio=$PSI_AUDIO_ONLY_TASK_WEIGHT \
|
| 229 |
+
--data.audio.sample-stride=1 \
|
| 230 |
+
--data.audio.motion-fps=50 \
|
| 231 |
+
--data.audio.audio-sampling-rate=16000 \
|
| 232 |
+
--data.audio.audio-window-seconds=1.0 \
|
| 233 |
+
--data.audio.audio-left-context-seconds=0.0 \
|
| 234 |
+
--data.audio.full-audio \
|
| 235 |
+
--data.audio-command.sources-json=$PSI_AUDIO_COMMAND_SOURCES_JSON \
|
| 236 |
+
--data.audio-command-ratio=$PSI_AUDIO_COMMAND_TEXT_TASK_WEIGHT \
|
| 237 |
+
--data.audio-command-only-ratio=$PSI_AUDIO_COMMAND_ONLY_TASK_WEIGHT \
|
| 238 |
+
--data.audio-command.sample-stride=1 \
|
| 239 |
+
--data.audio-command.motion-fps=50 \
|
| 240 |
+
--data.audio-command.audio-sampling-rate=16000 \
|
| 241 |
+
--data.audio-command.audio-window-seconds=1.0 \
|
| 242 |
+
--data.audio-command.audio-left-context-seconds=0.0 \
|
| 243 |
+
--data.audio-command.full-audio \
|
| 244 |
+
--data.video-ratio=$PSI_VIDEO_TEXT_TASK_WEIGHT \
|
| 245 |
+
--data.video-only-ratio=$PSI_VIDEO_ONLY_TASK_WEIGHT \
|
| 246 |
+
--data.video.sample-stride=1 \
|
| 247 |
+
--data.video.motion-fps=50 \
|
| 248 |
+
--data.video.video-sample-fps=$VIDEO_SAMPLE_FPS \
|
| 249 |
+
--data.video.video-size-pixels=$VIDEO_SIZE_PIXELS \
|
| 250 |
+
--data.video.video-decoder=$VIDEO_DECODER \
|
| 251 |
+
--data.video.video-cache-size=$VIDEO_CACHE_SIZE \
|
| 252 |
+
--data.video.sources-json=$PSI_VIDEO_SOURCES_JSON \
|
| 253 |
+
--data.video-clip-text-ratio=$PSI_VIDEO_CLIP_TEXT_TASK_WEIGHT \
|
| 254 |
+
--data.video-clip-only-ratio=$PSI_VIDEO_CLIP_ONLY_TASK_WEIGHT \
|
| 255 |
+
--data.video-clip.sources-json=$PSI_VIDEO_CLIP_SOURCES_JSON \
|
| 256 |
+
--data.video-clip.sample-stride=1 \
|
| 257 |
+
--data.video-clip.motion-fps=50 \
|
| 258 |
+
--data.video-clip.video-sample-fps=$VIDEO_CLIP_SAMPLE_FPS \
|
| 259 |
+
--data.video-clip.video-size-pixels=$VIDEO_SIZE_PIXELS \
|
| 260 |
+
--data.video-clip.video-decoder=$VIDEO_DECODER \
|
| 261 |
+
--data.video-clip.video-cache-size=$VIDEO_CACHE_SIZE \
|
| 262 |
+
--data.video-clip.motion-cache-size=$VIDEO_CLIP_MOTION_CACHE_SIZE \
|
| 263 |
+
$VIDEO_CLIP_TRUST_METADATA_FLAG \
|
| 264 |
+
--data.video-clip.records-cache-dir=$VIDEO_CLIP_RECORDS_CACHE_DIR \
|
| 265 |
+
--data.video-clip.action-exec-horizon=$ACTION_EXEC_HORIZON \
|
| 266 |
+
--data.sampler=batch_mixture \
|
| 267 |
+
--data.val-task=text,audio,audio_only,audio_command,audio_command_only,video_clip,video_clip_only \
|
| 268 |
+
--data.transform.repack.num-past-frames=$NUM_PAST_FRAMES \
|
| 269 |
+
--data.transform.repack.textop-actions \
|
| 270 |
+
--data.transform.repack.action-chunk-size=$ACTION_CHUNK_SIZE \
|
| 271 |
+
--data.transform.repack.pad-action-dim=39 \
|
| 272 |
+
--data.transform.repack.pad-state-dim=39 \
|
| 273 |
+
--data.transform.field.action_norm_type=bounds_q99 \
|
| 274 |
+
--data.transform.field.stat-path=assets/stats/seed_all_g1_motion39_textop_stats.json \
|
| 275 |
+
--data.transform.field.normalize-state \
|
| 276 |
+
--data.transform.field.pad-action-dim=39 \
|
| 277 |
+
--data.transform.field.pad-state-dim=39 \
|
| 278 |
+
--data.transform.model.resize.size 240 320 \
|
| 279 |
+
--data.transform.model.center_crop.size 240 320 \
|
| 280 |
+
--data.transform.model.no-img-aug \
|
| 281 |
+
--data.transform.model.text-only \
|
| 282 |
+
--data.transform.model.audio-only \
|
| 283 |
+
--data.transform.model.video-only \
|
| 284 |
+
--data.transform.model.video-size-pixels=$VIDEO_SIZE_PIXELS \
|
| 285 |
+
--model.model_name_or_path=$PSI_PRETRAIN_BACKBONE_PATH \
|
| 286 |
+
--model.noise-scheduler=flow \
|
| 287 |
+
--model.n_conditions=0 \
|
| 288 |
+
--model.action-chunk-size=$ACTION_CHUNK_SIZE \
|
| 289 |
+
--model.action-dim=39 \
|
| 290 |
+
--model.action-exec-horizon=$ACTION_EXEC_HORIZON \
|
| 291 |
+
--model.observation-horizon=$STATE_HORIZON \
|
| 292 |
+
--model.odim=39 \
|
| 293 |
+
--model.view_feature_dim=2048 \
|
| 294 |
+
--model.hidden-dim=$ACTION_HIDDEN_DIM \
|
| 295 |
+
--model.num-blocks=$ACTION_NUM_BLOCKS \
|
| 296 |
+
--model.nhead=$ACTION_NHEADS \
|
| 297 |
+
--model.use-future-progress-embedding \
|
| 298 |
+
--model.future-progress-dim=$PROGRESS_DIM \
|
| 299 |
+
--model.tune-vlm \
|
| 300 |
+
--model.lang-backbone-lr=1e-5 \
|
| 301 |
+
--model.no-use_film \
|
| 302 |
+
--model.no-combined_temb \
|
| 303 |
+
--model.rtc \
|
| 304 |
+
--model.max-delay=11 \
|
| 305 |
+
--train.max-grad-value=1 \
|
| 306 |
+
--train.loss-group-weight.joint-abs=1.0 \
|
| 307 |
+
--train.loss-group-weight.root-delta-xy=3.0 \
|
| 308 |
+
--train.loss-group-weight.root-rot5-delta=10.0 \
|
| 309 |
+
--train.loss-group-weight.root-height=15.0 \
|
| 310 |
+
--train.loss-group-weight.foot-contact=0.1
|
| 311 |
+
"
|
| 312 |
+
torchrun --nproc_per_node=$NPROC_PER_NODE --master_port=29500 scripts/train.py \
|
| 313 |
+
$args "$@"
|
posttrain-v1-psi0-all-7task-tune-vlm-direct-dense-actionhead-perfect-resume.sh
ADDED
|
@@ -0,0 +1,469 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#!/usr/bin/env bash
|
| 2 |
+
|
| 3 |
+
set -euo pipefail
|
| 4 |
+
|
| 5 |
+
pkill -f monitor.py || true
|
| 6 |
+
|
| 7 |
+
DEFAULT_RESUME_TARGET=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/Psi0-yth-v18/runs/posttrain_v1_trimodal/posttrain.sg.flow1000.cwml.lr1.0e-04.b256.gpus8.2606041447
|
| 8 |
+
RESUME_TARGET=${1:-${RESUME_TARGET:-$DEFAULT_RESUME_TARGET}}
|
| 9 |
+
if [ "$#" -ge 1 ]; then
|
| 10 |
+
shift
|
| 11 |
+
fi
|
| 12 |
+
|
| 13 |
+
cd /cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/Psi0-yth-v21
|
| 14 |
+
export PYTHONPATH=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/Psi0-yth-v21/src
|
| 15 |
+
export CUDA_VISIBLE_DEVICES=${CUDA_VISIBLE_DEVICES:-0,1,2,3,4,5,6,7}
|
| 16 |
+
export PSI_OUTPUT_DIR=${PSI_OUTPUT_DIR:-/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/Psi0-yth-v21/runs}
|
| 17 |
+
export PSI_LOG_REPORT_TO=wandb
|
| 18 |
+
|
| 19 |
+
source /cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/Psi0-yth-v15/.venv-psi/bin/activate
|
| 20 |
+
[ -f .env ] && source .env
|
| 21 |
+
|
| 22 |
+
export WANDB_API_KEY="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"
|
| 23 |
+
|
| 24 |
+
export OMP_NUM_THREADS=8
|
| 25 |
+
export TF_CPP_MIN_LOG_LEVEL=3
|
| 26 |
+
|
| 27 |
+
# Keep JSON sources readable in this script, but pass compact JSON to tyro.
|
| 28 |
+
compact_json() {
|
| 29 |
+
python -c 'import json, sys; print(json.dumps(json.load(sys.stdin), separators=(",", ":")))'
|
| 30 |
+
}
|
| 31 |
+
|
| 32 |
+
resolve_resume_paths() {
|
| 33 |
+
python - "$1" <<'PY'
|
| 34 |
+
import os
|
| 35 |
+
import re
|
| 36 |
+
import sys
|
| 37 |
+
|
| 38 |
+
target = os.path.abspath(sys.argv[1])
|
| 39 |
+
|
| 40 |
+
def latest_numeric_ckpt(checkpoint_dir: str) -> str:
|
| 41 |
+
ckpts = []
|
| 42 |
+
for name in os.listdir(checkpoint_dir):
|
| 43 |
+
m = re.fullmatch(r"ckpt_(\d+)", name)
|
| 44 |
+
if m:
|
| 45 |
+
ckpts.append((int(m.group(1)), os.path.join(checkpoint_dir, name)))
|
| 46 |
+
if not ckpts:
|
| 47 |
+
raise SystemExit(f"No numeric checkpoint found under {checkpoint_dir}")
|
| 48 |
+
ckpts.sort()
|
| 49 |
+
return ckpts[-1][1]
|
| 50 |
+
|
| 51 |
+
if os.path.exists(os.path.join(target, "model.safetensors")):
|
| 52 |
+
ckpt_dir = target
|
| 53 |
+
run_dir = os.path.dirname(os.path.dirname(target))
|
| 54 |
+
elif os.path.exists(os.path.join(target, "run_config.json")) and os.path.isdir(os.path.join(target, "checkpoints")):
|
| 55 |
+
run_dir = target
|
| 56 |
+
ckpt_dir = latest_numeric_ckpt(os.path.join(run_dir, "checkpoints"))
|
| 57 |
+
else:
|
| 58 |
+
raise SystemExit(
|
| 59 |
+
"RESUME_TARGET must be either a run dir containing run_config.json/checkpoints "
|
| 60 |
+
"or an exact checkpoint dir containing model.safetensors"
|
| 61 |
+
)
|
| 62 |
+
|
| 63 |
+
print(run_dir)
|
| 64 |
+
print(ckpt_dir)
|
| 65 |
+
PY
|
| 66 |
+
}
|
| 67 |
+
|
| 68 |
+
mapfile -t _resume_info < <(resolve_resume_paths "$RESUME_TARGET")
|
| 69 |
+
RUN_DIR=${_resume_info[0]}
|
| 70 |
+
RESUME_CKPT_DIR=${_resume_info[1]}
|
| 71 |
+
RUN_CONFIG_JSON=$RUN_DIR/run_config.json
|
| 72 |
+
|
| 73 |
+
if [ ! -f "$RUN_CONFIG_JSON" ]; then
|
| 74 |
+
echo "Missing run_config.json: $RUN_CONFIG_JSON"
|
| 75 |
+
exit 1
|
| 76 |
+
fi
|
| 77 |
+
|
| 78 |
+
eval "$(
|
| 79 |
+
python - "$RUN_CONFIG_JSON" <<'PY'
|
| 80 |
+
import json
|
| 81 |
+
import re
|
| 82 |
+
import shlex
|
| 83 |
+
import sys
|
| 84 |
+
|
| 85 |
+
cfg = json.load(open(sys.argv[1]))
|
| 86 |
+
|
| 87 |
+
wandb_cfg = cfg.get("wandb") or {}
|
| 88 |
+
train_cfg = cfg.get("train") or {}
|
| 89 |
+
model_cfg = cfg.get("model") or {}
|
| 90 |
+
env_cfg = cfg.get("environment_variables") or {}
|
| 91 |
+
|
| 92 |
+
run_name = wandb_cfg.get("name") or ""
|
| 93 |
+
world_size = env_cfg.get("WORLD_SIZE") or ""
|
| 94 |
+
if not world_size:
|
| 95 |
+
m = re.search(r"\.gpus(\d+)\.", run_name)
|
| 96 |
+
if m:
|
| 97 |
+
world_size = m.group(1)
|
| 98 |
+
|
| 99 |
+
values = {
|
| 100 |
+
"OLD_TIMESTAMP": str(cfg["timestamp"]),
|
| 101 |
+
"OLD_WANDB_RUN_ID": str(wandb_cfg.get("id") or ""),
|
| 102 |
+
"OLD_EXP_NAME": str(cfg["exp"]),
|
| 103 |
+
"OLD_TRAIN_NAME": str(train_cfg["name"]),
|
| 104 |
+
"OLD_OUTPUT_DIR": str(train_cfg["output_dir"]),
|
| 105 |
+
"OLD_RUN_NAME": str(run_name),
|
| 106 |
+
"EXPECTED_WORLD_SIZE": str(world_size),
|
| 107 |
+
"OLD_TRAIN_BATCH_SIZE": str(train_cfg["train_batch_size"]),
|
| 108 |
+
"OLD_VAL_BATCH_SIZE": str(train_cfg["val_batch_size"]),
|
| 109 |
+
"OLD_MAX_TRAINING_STEPS": str(train_cfg["max_training_steps"]),
|
| 110 |
+
"OLD_HIDDEN_DIM": str(model_cfg["hidden_dim"]),
|
| 111 |
+
"OLD_NUM_BLOCKS": str(model_cfg["num_blocks"]),
|
| 112 |
+
"OLD_NHEAD": str(model_cfg["nhead"]),
|
| 113 |
+
}
|
| 114 |
+
|
| 115 |
+
for k, v in values.items():
|
| 116 |
+
print(f"{k}={shlex.quote(v)}")
|
| 117 |
+
PY
|
| 118 |
+
)"
|
| 119 |
+
|
| 120 |
+
RUN_TIMESTAMP=${PSI_RESUME_TIMESTAMP:-$(date +"%y%m%d%H%M")}
|
| 121 |
+
EXP_NAME=${PSI_RESUME_EXP:-$OLD_EXP_NAME}
|
| 122 |
+
TRAIN_NAME=${PSI_RESUME_TRAIN_NAME:-$OLD_TRAIN_NAME}
|
| 123 |
+
TRAIN_BATCH_SIZE=$OLD_TRAIN_BATCH_SIZE
|
| 124 |
+
VAL_BATCH_SIZE=$OLD_VAL_BATCH_SIZE
|
| 125 |
+
MODEL_HIDDEN_DIM=$OLD_HIDDEN_DIM
|
| 126 |
+
MODEL_NUM_BLOCKS=$OLD_NUM_BLOCKS
|
| 127 |
+
MODEL_NHEADS=$OLD_NHEAD
|
| 128 |
+
MODEL_HEAD_DIM=$((MODEL_HIDDEN_DIM / MODEL_NHEADS))
|
| 129 |
+
MAX_TRAINING_STEPS=${PSI_MAX_TRAINING_STEPS:-1500000}
|
| 130 |
+
|
| 131 |
+
case "$(basename "$RESUME_CKPT_DIR")" in
|
| 132 |
+
ckpt_*) RESUME_INITIAL_STEP=$(( ${RESUME_CKPT_DIR##*_} + 1 )) ;;
|
| 133 |
+
best_ckpt)
|
| 134 |
+
RESUME_INITIAL_STEP=$(python - "$RUN_DIR/checkpoints/best_eval_metric.json" <<'PY'
|
| 135 |
+
import json, sys
|
| 136 |
+
print(int(json.load(open(sys.argv[1]))["global_step"]) + 1)
|
| 137 |
+
PY
|
| 138 |
+
)
|
| 139 |
+
;;
|
| 140 |
+
*) RESUME_INITIAL_STEP=0 ;;
|
| 141 |
+
esac
|
| 142 |
+
|
| 143 |
+
if [ "$MAX_TRAINING_STEPS" -le "$RESUME_INITIAL_STEP" ]; then
|
| 144 |
+
echo "PSI_MAX_TRAINING_STEPS=$MAX_TRAINING_STEPS must be > resume initial step $RESUME_INITIAL_STEP"
|
| 145 |
+
exit 1
|
| 146 |
+
fi
|
| 147 |
+
|
| 148 |
+
if [ "${RESUME_WANDB_ORIGINAL:-0}" = "1" ]; then
|
| 149 |
+
if [ -z "$OLD_WANDB_RUN_ID" ]; then
|
| 150 |
+
echo "Old run has no wandb id in $RUN_CONFIG_JSON, cannot resume original wandb run."
|
| 151 |
+
exit 1
|
| 152 |
+
fi
|
| 153 |
+
export WANDB_RUN_ID=$OLD_WANDB_RUN_ID
|
| 154 |
+
export WANDB_RESUME=must
|
| 155 |
+
WANDB_ARGS="--wandb.id=$OLD_WANDB_RUN_ID --wandb.resume=must"
|
| 156 |
+
else
|
| 157 |
+
unset WANDB_RUN_ID
|
| 158 |
+
export WANDB_RESUME=never
|
| 159 |
+
WANDB_ARGS="--wandb.resume=never"
|
| 160 |
+
fi
|
| 161 |
+
|
| 162 |
+
PSI_SEED_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/Psi0/dataset/seed_all
|
| 163 |
+
PSI_SEED_METADATA=metadata/seed_metadata_v004.csv
|
| 164 |
+
|
| 165 |
+
PSI_DANCE_MOTION_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/GR00T-WholeBodyControl/gear_sonic_deploy/reference/converted/psi0_dance/gvhmr
|
| 166 |
+
PSI_DANCE_AUDIO_VIDEO_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/dance_1723_video_psi0/dance_1723_video_10s
|
| 167 |
+
PSI_DANCE_VIDEO_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/dance_1723_video_psi0/dance_1723_video_10s
|
| 168 |
+
PSI_DANCE_VIDEO_EMBED_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/dance_1723_video_psi0/dance_1723_video_10s_embeds
|
| 169 |
+
PSI_VIDEO_DEMO_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/videogen/motion_example_0506_demo
|
| 170 |
+
PSI_2264_WITH_AUDIO_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/videogen/2264_with_audio
|
| 171 |
+
PSI_QWEN3_TTS_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/qwen3-tts
|
| 172 |
+
PSI_VIDEO_CLIP_DATASET_ROOT=${PSI_VIDEO_CLIP_DATASET_ROOT:-/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/videogen/psi0_video_clip_50f_dense_stride1_direct_full}
|
| 173 |
+
PSI_DANCE_VIDEO_CLIP_ROOT=$PSI_VIDEO_CLIP_DATASET_ROOT/dance_1723_video_10s
|
| 174 |
+
PSI_MOTION_EXAMPLE_VIDEO_CLIP_ROOT=$PSI_VIDEO_CLIP_DATASET_ROOT/motion_example_0506_demo
|
| 175 |
+
PSI_2264_WITH_AUDIO_VIDEO_CLIP_ROOT=$PSI_VIDEO_CLIP_DATASET_ROOT/2264_with_audio
|
| 176 |
+
PSI_DANCE_VIDEO_CLIP_EMBED_ROOT=${PSI_DANCE_VIDEO_CLIP_EMBED_ROOT:-$PSI_DANCE_VIDEO_CLIP_ROOT/embeds}
|
| 177 |
+
PSI_MOTION_EXAMPLE_VIDEO_CLIP_EMBED_ROOT=${PSI_MOTION_EXAMPLE_VIDEO_CLIP_EMBED_ROOT:-$PSI_MOTION_EXAMPLE_VIDEO_CLIP_ROOT/embeds}
|
| 178 |
+
PSI_2264_WITH_AUDIO_VIDEO_CLIP_EMBED_ROOT=${PSI_2264_WITH_AUDIO_VIDEO_CLIP_EMBED_ROOT:-$PSI_2264_WITH_AUDIO_VIDEO_CLIP_ROOT/embeds}
|
| 179 |
+
PSI_MOTION_EXAMPLE_VIDEO_EMBED_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/videogen/motion_example_0506_demo_video_embeds
|
| 180 |
+
|
| 181 |
+
if [ -z "${PSI_AUDIO_SOURCES_JSON:-}" ]; then
|
| 182 |
+
PSI_AUDIO_SOURCES_JSON=$(compact_json <<JSON
|
| 183 |
+
[
|
| 184 |
+
{
|
| 185 |
+
"name": "psi0_dance_audio",
|
| 186 |
+
"root_dir": "$PSI_DANCE_MOTION_ROOT",
|
| 187 |
+
"video_dir": "$PSI_DANCE_AUDIO_VIDEO_ROOT",
|
| 188 |
+
"dataset_name": "seed-audio-g1"
|
| 189 |
+
},
|
| 190 |
+
{
|
| 191 |
+
"name": "2264_with_audio",
|
| 192 |
+
"dataset_root": "$PSI_2264_WITH_AUDIO_ROOT",
|
| 193 |
+
"metadata_jsonl": "metadata_convert.jsonl",
|
| 194 |
+
"motion_dir_key": "converted_path",
|
| 195 |
+
"audio_path_key": "video_path",
|
| 196 |
+
"dataset_name": "2264-with-audio-g1"
|
| 197 |
+
}
|
| 198 |
+
]
|
| 199 |
+
JSON
|
| 200 |
+
)
|
| 201 |
+
fi
|
| 202 |
+
|
| 203 |
+
if [ -z "${PSI_AUDIO_COMMAND_SOURCES_JSON:-}" ]; then
|
| 204 |
+
PSI_AUDIO_COMMAND_SOURCES_JSON=$(compact_json <<JSON
|
| 205 |
+
[
|
| 206 |
+
{
|
| 207 |
+
"name": "qwen3_tts_seed_audio_command",
|
| 208 |
+
"root_dir": "$PSI_SEED_ROOT",
|
| 209 |
+
"dataset_root": "$PSI_QWEN3_TTS_ROOT",
|
| 210 |
+
"metadata_jsonl": "metadata.jsonl",
|
| 211 |
+
"motion_dir_key": "filename",
|
| 212 |
+
"audio_path_key": "audio_path",
|
| 213 |
+
"instruction_key": "content_short_description",
|
| 214 |
+
"motion_metadata_csv": "$PSI_SEED_METADATA",
|
| 215 |
+
"motion_metadata_key": "filename",
|
| 216 |
+
"motion_metadata_path_key": "move_g1_path",
|
| 217 |
+
"audio_file_extensions": ".wav",
|
| 218 |
+
"audio_task": "command",
|
| 219 |
+
"dataset_name": "seed-audio-command-g1"
|
| 220 |
+
}
|
| 221 |
+
]
|
| 222 |
+
JSON
|
| 223 |
+
)
|
| 224 |
+
fi
|
| 225 |
+
|
| 226 |
+
if [ -z "${PSI_VIDEO_SOURCES_JSON:-}" ]; then
|
| 227 |
+
PSI_VIDEO_SOURCES_JSON=$(compact_json <<JSON
|
| 228 |
+
[
|
| 229 |
+
{
|
| 230 |
+
"name": "psi0_dance_gvhmr",
|
| 231 |
+
"root_dir": "$PSI_DANCE_MOTION_ROOT",
|
| 232 |
+
"video_dir": "$PSI_DANCE_VIDEO_ROOT",
|
| 233 |
+
"video_embed_dir": "$PSI_DANCE_VIDEO_EMBED_ROOT",
|
| 234 |
+
"dataset_name": "seed-video-g1"
|
| 235 |
+
},
|
| 236 |
+
{
|
| 237 |
+
"name": "motion_example_0506_demo",
|
| 238 |
+
"dataset_root": "$PSI_VIDEO_DEMO_ROOT",
|
| 239 |
+
"metadata_jsonl": "metadata.jsonl",
|
| 240 |
+
"motion_dir_key": "converted_path",
|
| 241 |
+
"video_path_key": "video_path",
|
| 242 |
+
"instruction_key": "video_caption",
|
| 243 |
+
"video_embed_dir": "$PSI_MOTION_EXAMPLE_VIDEO_EMBED_ROOT",
|
| 244 |
+
"dataset_name": "motion-example-video-g1"
|
| 245 |
+
}
|
| 246 |
+
]
|
| 247 |
+
JSON
|
| 248 |
+
)
|
| 249 |
+
fi
|
| 250 |
+
|
| 251 |
+
if [ -z "${PSI_VIDEO_CLIP_SOURCES_JSON:-}" ]; then
|
| 252 |
+
PSI_VIDEO_CLIP_SOURCES_JSON=$(compact_json <<JSON
|
| 253 |
+
[
|
| 254 |
+
{
|
| 255 |
+
"name": "dance_1723_video_10s_clip50",
|
| 256 |
+
"dataset_root": "$PSI_DANCE_VIDEO_CLIP_ROOT",
|
| 257 |
+
"metadata_jsonl": "metadata.jsonl",
|
| 258 |
+
"motion_dir_key": "converted_path",
|
| 259 |
+
"video_path_key": "video_path",
|
| 260 |
+
"instruction_key": "video_caption",
|
| 261 |
+
"video_embed_dir": "$PSI_DANCE_VIDEO_CLIP_EMBED_ROOT",
|
| 262 |
+
"dataset_name": "dance-video-clip-g1"
|
| 263 |
+
},
|
| 264 |
+
{
|
| 265 |
+
"name": "motion_example_0506_demo_clip50",
|
| 266 |
+
"dataset_root": "$PSI_MOTION_EXAMPLE_VIDEO_CLIP_ROOT",
|
| 267 |
+
"metadata_jsonl": "metadata.jsonl",
|
| 268 |
+
"motion_dir_key": "converted_path",
|
| 269 |
+
"video_path_key": "video_path",
|
| 270 |
+
"instruction_key": "video_caption",
|
| 271 |
+
"video_embed_dir": "$PSI_MOTION_EXAMPLE_VIDEO_CLIP_EMBED_ROOT",
|
| 272 |
+
"dataset_name": "motion-example-video-clip-g1"
|
| 273 |
+
},
|
| 274 |
+
{
|
| 275 |
+
"name": "2264_with_audio_clip50",
|
| 276 |
+
"dataset_root": "$PSI_2264_WITH_AUDIO_VIDEO_CLIP_ROOT",
|
| 277 |
+
"metadata_jsonl": "metadata.jsonl",
|
| 278 |
+
"motion_dir_key": "converted_path",
|
| 279 |
+
"video_path_key": "video_path",
|
| 280 |
+
"instruction_key": "video_caption",
|
| 281 |
+
"video_embed_dir": "$PSI_2264_WITH_AUDIO_VIDEO_CLIP_EMBED_ROOT",
|
| 282 |
+
"dataset_name": "2264-with-audio-video-clip-g1"
|
| 283 |
+
}
|
| 284 |
+
]
|
| 285 |
+
JSON
|
| 286 |
+
)
|
| 287 |
+
fi
|
| 288 |
+
|
| 289 |
+
PSI_TEXT_TASK_WEIGHT=1.0
|
| 290 |
+
PSI_AUDIO_TEXT_TASK_WEIGHT=3.8509
|
| 291 |
+
PSI_AUDIO_ONLY_TASK_WEIGHT=3.8509
|
| 292 |
+
PSI_AUDIO_COMMAND_TEXT_TASK_WEIGHT=5.7921
|
| 293 |
+
PSI_AUDIO_COMMAND_ONLY_TASK_WEIGHT=5.7921
|
| 294 |
+
PSI_VIDEO_TEXT_TASK_WEIGHT=0
|
| 295 |
+
PSI_VIDEO_ONLY_TASK_WEIGHT=0
|
| 296 |
+
PSI_VIDEO_CLIP_TEXT_TASK_WEIGHT=1.7881
|
| 297 |
+
PSI_VIDEO_CLIP_ONLY_TASK_WEIGHT=1.7881
|
| 298 |
+
|
| 299 |
+
PSI_PRETRAIN_BACKBONE_PATH=/cpfs01/VideoGen/public_ckpt/Qwen/Qwen2.5-Omni-3B
|
| 300 |
+
NUM_PAST_FRAMES=0
|
| 301 |
+
STATE_HORIZON=$((NUM_PAST_FRAMES + 1))
|
| 302 |
+
ACTION_CHUNK_SIZE=30
|
| 303 |
+
ACTION_EXEC_HORIZON=20
|
| 304 |
+
PROGRESS_DIM=5
|
| 305 |
+
VIDEO_SAMPLE_FPS=1.0
|
| 306 |
+
VIDEO_CLIP_SAMPLE_FPS=8.0
|
| 307 |
+
VIDEO_SIZE_PIXELS=50176
|
| 308 |
+
VIDEO_DECODER=ffmpeg
|
| 309 |
+
VIDEO_CACHE_SIZE=64
|
| 310 |
+
VIDEO_CLIP_MOTION_CACHE_SIZE=256
|
| 311 |
+
VIDEO_CLIP_TRUST_METADATA=${VIDEO_CLIP_TRUST_METADATA:-1}
|
| 312 |
+
VIDEO_CLIP_RECORDS_CACHE_DIR=${VIDEO_CLIP_RECORDS_CACHE_DIR:-$PSI_VIDEO_CLIP_DATASET_ROOT/.index_cache}
|
| 313 |
+
if [ "$VIDEO_CLIP_TRUST_METADATA" = "1" ] || [ "$VIDEO_CLIP_TRUST_METADATA" = "true" ] || [ "$VIDEO_CLIP_TRUST_METADATA" = "True" ]; then
|
| 314 |
+
VIDEO_CLIP_TRUST_METADATA_FLAG="--data.video-clip.trust-metadata"
|
| 315 |
+
else
|
| 316 |
+
VIDEO_CLIP_TRUST_METADATA_FLAG="--data.video-clip.no-trust-metadata"
|
| 317 |
+
fi
|
| 318 |
+
# DATALOADER_NUM_WORKERS=4
|
| 319 |
+
# VAL_DATALOADER_NUM_WORKERS=2
|
| 320 |
+
|
| 321 |
+
DATALOADER_NUM_WORKERS=${DATALOADER_NUM_WORKERS:-0}
|
| 322 |
+
VAL_DATALOADER_NUM_WORKERS=${VAL_DATALOADER_NUM_WORKERS:-0}
|
| 323 |
+
|
| 324 |
+
NPROC_PER_NODE=$(echo "$CUDA_VISIBLE_DEVICES" | tr ',' '\n' | wc -l | awk '{print $1}')
|
| 325 |
+
if [ -n "$EXPECTED_WORLD_SIZE" ] && [ "$NPROC_PER_NODE" != "$EXPECTED_WORLD_SIZE" ]; then
|
| 326 |
+
echo "GPU count mismatch: old run used $EXPECTED_WORLD_SIZE GPUs, current CUDA_VISIBLE_DEVICES gives $NPROC_PER_NODE"
|
| 327 |
+
exit 1
|
| 328 |
+
fi
|
| 329 |
+
|
| 330 |
+
MASTER_PORT=${MASTER_PORT:-29500}
|
| 331 |
+
|
| 332 |
+
echo "Training with $NPROC_PER_NODE GPU(s): $CUDA_VISIBLE_DEVICES"
|
| 333 |
+
echo "Resume run dir: $RUN_DIR"
|
| 334 |
+
echo "Resume checkpoint dir: $RESUME_CKPT_DIR"
|
| 335 |
+
echo "Resume initial step: $RESUME_INITIAL_STEP"
|
| 336 |
+
echo "Old run name: $OLD_RUN_NAME"
|
| 337 |
+
echo "Old wandb id: $OLD_WANDB_RUN_ID"
|
| 338 |
+
echo "Wandb mode: ${RESUME_WANDB_ORIGINAL:-0} (0=new run, 1=resume old run)"
|
| 339 |
+
echo "Run timestamp: $RUN_TIMESTAMP"
|
| 340 |
+
echo "Exp: $EXP_NAME"
|
| 341 |
+
echo "Model scale config: hidden=$MODEL_HIDDEN_DIM depth=$MODEL_NUM_BLOCKS heads=$MODEL_NHEADS head_dim=$MODEL_HEAD_DIM"
|
| 342 |
+
echo "Max training steps: $MAX_TRAINING_STEPS"
|
| 343 |
+
echo "Output dir: $PSI_OUTPUT_DIR"
|
| 344 |
+
echo "2264 with audio root: $PSI_2264_WITH_AUDIO_ROOT"
|
| 345 |
+
echo "Audio sources: $PSI_AUDIO_SOURCES_JSON"
|
| 346 |
+
echo "Video clip sources: $PSI_VIDEO_CLIP_SOURCES_JSON"
|
| 347 |
+
|
| 348 |
+
args="posttrain_v1_seed_text_audio_video_mix_psi0_config \
|
| 349 |
+
--seed=292285 \
|
| 350 |
+
--exp=$EXP_NAME \
|
| 351 |
+
--timestamp=$RUN_TIMESTAMP \
|
| 352 |
+
--train.name=$TRAIN_NAME \
|
| 353 |
+
--train.data_parallel=ddp \
|
| 354 |
+
--train.mixed_precision=bf16 \
|
| 355 |
+
--train.train_batch_size=$TRAIN_BATCH_SIZE \
|
| 356 |
+
--train.val_batch_size=$VAL_BATCH_SIZE \
|
| 357 |
+
--train.num-workers=$DATALOADER_NUM_WORKERS \
|
| 358 |
+
--train.val-num-workers=$VAL_DATALOADER_NUM_WORKERS \
|
| 359 |
+
--train.resume_from_checkpoint=$RESUME_CKPT_DIR \
|
| 360 |
+
--train.resume_load_full_state \
|
| 361 |
+
--train.max_checkpoints_to_keep=1 \
|
| 362 |
+
--train.gradient_accumulation_steps=1 \
|
| 363 |
+
--train.learning_rate=1e-4 \
|
| 364 |
+
--train.max_training_steps=$MAX_TRAINING_STEPS \
|
| 365 |
+
--train.warmup_ratio=None \
|
| 366 |
+
--train.warmup_steps=1000 \
|
| 367 |
+
--train.checkpointing_steps=1000 \
|
| 368 |
+
--train.validation_steps=1000 \
|
| 369 |
+
--train.val_num_batches=20 \
|
| 370 |
+
--train.max_grad_norm=1.0 \
|
| 371 |
+
--train.lr_scheduler_type=cosine_with_min_lr \
|
| 372 |
+
--train.lr_scheduler_kwargs.betas 0.9 0.999 \
|
| 373 |
+
--train.lr_scheduler_kwargs.weight_decay=0 \
|
| 374 |
+
--train.lr_scheduler_kwargs.eps=1e-8 \
|
| 375 |
+
--log.report_to=$PSI_LOG_REPORT_TO \
|
| 376 |
+
$WANDB_ARGS \
|
| 377 |
+
--train.output_dir=$PSI_OUTPUT_DIR \
|
| 378 |
+
--data.text.root-dir=$PSI_SEED_ROOT \
|
| 379 |
+
--data.text.metadata-csv=$PSI_SEED_METADATA \
|
| 380 |
+
--data.text-ratio=$PSI_TEXT_TASK_WEIGHT \
|
| 381 |
+
--data.audio.sources-json=$PSI_AUDIO_SOURCES_JSON \
|
| 382 |
+
--data.audio-ratio=$PSI_AUDIO_TEXT_TASK_WEIGHT \
|
| 383 |
+
--data.audio-only-ratio=$PSI_AUDIO_ONLY_TASK_WEIGHT \
|
| 384 |
+
--data.audio.sample-stride=1 \
|
| 385 |
+
--data.audio.motion-fps=50 \
|
| 386 |
+
--data.audio.audio-sampling-rate=16000 \
|
| 387 |
+
--data.audio.audio-window-seconds=1.0 \
|
| 388 |
+
--data.audio.audio-left-context-seconds=0.0 \
|
| 389 |
+
--data.audio.full-audio \
|
| 390 |
+
--data.audio-command.sources-json=$PSI_AUDIO_COMMAND_SOURCES_JSON \
|
| 391 |
+
--data.audio-command-ratio=$PSI_AUDIO_COMMAND_TEXT_TASK_WEIGHT \
|
| 392 |
+
--data.audio-command-only-ratio=$PSI_AUDIO_COMMAND_ONLY_TASK_WEIGHT \
|
| 393 |
+
--data.audio-command.sample-stride=1 \
|
| 394 |
+
--data.audio-command.motion-fps=50 \
|
| 395 |
+
--data.audio-command.audio-sampling-rate=16000 \
|
| 396 |
+
--data.audio-command.audio-window-seconds=1.0 \
|
| 397 |
+
--data.audio-command.audio-left-context-seconds=0.0 \
|
| 398 |
+
--data.audio-command.full-audio \
|
| 399 |
+
--data.video-ratio=$PSI_VIDEO_TEXT_TASK_WEIGHT \
|
| 400 |
+
--data.video-only-ratio=$PSI_VIDEO_ONLY_TASK_WEIGHT \
|
| 401 |
+
--data.video.sample-stride=1 \
|
| 402 |
+
--data.video.motion-fps=50 \
|
| 403 |
+
--data.video.video-sample-fps=$VIDEO_SAMPLE_FPS \
|
| 404 |
+
--data.video.video-size-pixels=$VIDEO_SIZE_PIXELS \
|
| 405 |
+
--data.video.video-decoder=$VIDEO_DECODER \
|
| 406 |
+
--data.video.video-cache-size=$VIDEO_CACHE_SIZE \
|
| 407 |
+
--data.video.sources-json=$PSI_VIDEO_SOURCES_JSON \
|
| 408 |
+
--data.video-clip-text-ratio=$PSI_VIDEO_CLIP_TEXT_TASK_WEIGHT \
|
| 409 |
+
--data.video-clip-only-ratio=$PSI_VIDEO_CLIP_ONLY_TASK_WEIGHT \
|
| 410 |
+
--data.video-clip.sources-json=$PSI_VIDEO_CLIP_SOURCES_JSON \
|
| 411 |
+
--data.video-clip.sample-stride=1 \
|
| 412 |
+
--data.video-clip.motion-fps=50 \
|
| 413 |
+
--data.video-clip.video-sample-fps=$VIDEO_CLIP_SAMPLE_FPS \
|
| 414 |
+
--data.video-clip.video-size-pixels=$VIDEO_SIZE_PIXELS \
|
| 415 |
+
--data.video-clip.video-decoder=$VIDEO_DECODER \
|
| 416 |
+
--data.video-clip.video-cache-size=$VIDEO_CACHE_SIZE \
|
| 417 |
+
--data.video-clip.motion-cache-size=$VIDEO_CLIP_MOTION_CACHE_SIZE \
|
| 418 |
+
$VIDEO_CLIP_TRUST_METADATA_FLAG \
|
| 419 |
+
--data.video-clip.records-cache-dir=$VIDEO_CLIP_RECORDS_CACHE_DIR \
|
| 420 |
+
--data.video-clip.action-exec-horizon=$ACTION_EXEC_HORIZON \
|
| 421 |
+
--data.sampler=batch_mixture \
|
| 422 |
+
--data.val-task=text,audio,audio_only,audio_command,audio_command_only,video_clip,video_clip_only \
|
| 423 |
+
--data.transform.repack.num-past-frames=$NUM_PAST_FRAMES \
|
| 424 |
+
--data.transform.repack.textop-actions \
|
| 425 |
+
--data.transform.repack.action-chunk-size=$ACTION_CHUNK_SIZE \
|
| 426 |
+
--data.transform.repack.pad-action-dim=39 \
|
| 427 |
+
--data.transform.repack.pad-state-dim=39 \
|
| 428 |
+
--data.transform.field.action_norm_type=bounds_q99 \
|
| 429 |
+
--data.transform.field.stat-path=assets/stats/seed_all_g1_motion39_textop_stats.json \
|
| 430 |
+
--data.transform.field.normalize-state \
|
| 431 |
+
--data.transform.field.pad-action-dim=39 \
|
| 432 |
+
--data.transform.field.pad-state-dim=39 \
|
| 433 |
+
--data.transform.model.resize.size 240 320 \
|
| 434 |
+
--data.transform.model.center_crop.size 240 320 \
|
| 435 |
+
--data.transform.model.no-img-aug \
|
| 436 |
+
--data.transform.model.text-only \
|
| 437 |
+
--data.transform.model.audio-only \
|
| 438 |
+
--data.transform.model.video-only \
|
| 439 |
+
--data.transform.model.video-size-pixels=$VIDEO_SIZE_PIXELS \
|
| 440 |
+
--model.model_name_or_path=$PSI_PRETRAIN_BACKBONE_PATH \
|
| 441 |
+
--model.noise-scheduler=flow \
|
| 442 |
+
--model.n_conditions=0 \
|
| 443 |
+
--model.action-chunk-size=$ACTION_CHUNK_SIZE \
|
| 444 |
+
--model.action-dim=39 \
|
| 445 |
+
--model.action-exec-horizon=$ACTION_EXEC_HORIZON \
|
| 446 |
+
--model.observation-horizon=$STATE_HORIZON \
|
| 447 |
+
--model.odim=39 \
|
| 448 |
+
--model.view_feature_dim=2048 \
|
| 449 |
+
--model.hidden-dim=$MODEL_HIDDEN_DIM \
|
| 450 |
+
--model.num-blocks=$MODEL_NUM_BLOCKS \
|
| 451 |
+
--model.nhead=$MODEL_NHEADS \
|
| 452 |
+
--model.use-future-progress-embedding \
|
| 453 |
+
--model.future-progress-dim=$PROGRESS_DIM \
|
| 454 |
+
--model.tune-vlm \
|
| 455 |
+
--model.lang-backbone-lr=1e-5 \
|
| 456 |
+
--model.no-use_film \
|
| 457 |
+
--model.no-combined_temb \
|
| 458 |
+
--model.rtc \
|
| 459 |
+
--model.max-delay=11 \
|
| 460 |
+
--train.max-grad-value=1 \
|
| 461 |
+
--train.loss-group-weight.joint-abs=1.0 \
|
| 462 |
+
--train.loss-group-weight.root-delta-xy=3.0 \
|
| 463 |
+
--train.loss-group-weight.root-rot5-delta=10.0 \
|
| 464 |
+
--train.loss-group-weight.root-height=15.0 \
|
| 465 |
+
--train.loss-group-weight.foot-contact=0.1
|
| 466 |
+
"
|
| 467 |
+
|
| 468 |
+
torchrun --nproc_per_node=$NPROC_PER_NODE --master_port=$MASTER_PORT scripts/train.py \
|
| 469 |
+
$args "$@"
|
posttrain-v1-psi0-all-7task-tune-vlm-direct-dense-full.sh
ADDED
|
@@ -0,0 +1,421 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
pkill -f monitor.py
|
| 2 |
+
|
| 3 |
+
cd /cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/Psi0-yth-v21
|
| 4 |
+
export PYTHONPATH=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/Psi0-yth-v21/src
|
| 5 |
+
export CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7
|
| 6 |
+
export PSI_OUTPUT_DIR=${PSI_OUTPUT_DIR:-/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/Psi0-yth-v21/runs}
|
| 7 |
+
export PSI_LOG_REPORT_TO=wandb
|
| 8 |
+
|
| 9 |
+
source /cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/Psi0-yth-v15/.venv-psi/bin/activate
|
| 10 |
+
[ -f .env ] && source .env
|
| 11 |
+
|
| 12 |
+
export WANDB_API_KEY="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"
|
| 13 |
+
|
| 14 |
+
export OMP_NUM_THREADS=8
|
| 15 |
+
export TF_CPP_MIN_LOG_LEVEL=3
|
| 16 |
+
|
| 17 |
+
# Keep JSON sources readable in this script, but pass compact JSON to tyro.
|
| 18 |
+
compact_json() {
|
| 19 |
+
python -c 'import json, sys; print(json.dumps(json.load(sys.stdin), separators=(",", ":")))'
|
| 20 |
+
}
|
| 21 |
+
|
| 22 |
+
PSI_SEED_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/Psi0/dataset/seed_all
|
| 23 |
+
PSI_SEED_METADATA=metadata/seed_metadata_v004.csv
|
| 24 |
+
PSI_TEXT_RECORDS_CACHE_DIR=${PSI_TEXT_RECORDS_CACHE_DIR:-$PSI_SEED_ROOT/.index_cache_multimodal_yth}
|
| 25 |
+
|
| 26 |
+
PSI_DANCE_MOTION_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/GR00T-WholeBodyControl/gear_sonic_deploy/reference/converted/psi0_dance/gvhmr
|
| 27 |
+
PSI_DANCE_AUDIO_VIDEO_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/dance_1723_video_psi0/dance_1723_video_10s
|
| 28 |
+
PSI_DANCE_VIDEO_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/dance_1723_video_psi0/dance_1723_video_10s
|
| 29 |
+
PSI_DANCE_VIDEO_EMBED_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/dance_1723_video_psi0/dance_1723_video_10s_embeds
|
| 30 |
+
PSI_VIDEO_DEMO_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/videogen/motion_example_0506_demo
|
| 31 |
+
PSI_2264_WITH_AUDIO_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/videogen/2264_with_audio
|
| 32 |
+
PSI_QWEN3_TTS_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/qwen3-tts
|
| 33 |
+
PSI_VIDEO_CLIP_DATASET_ROOT=${PSI_VIDEO_CLIP_DATASET_ROOT:-/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/videogen/psi0_video_clip_50f_dense_stride1_direct_full}
|
| 34 |
+
PSI_DANCE_VIDEO_CLIP_ROOT=$PSI_VIDEO_CLIP_DATASET_ROOT/dance_1723_video_10s
|
| 35 |
+
PSI_MOTION_EXAMPLE_VIDEO_CLIP_ROOT=$PSI_VIDEO_CLIP_DATASET_ROOT/motion_example_0506_demo
|
| 36 |
+
PSI_2264_WITH_AUDIO_VIDEO_CLIP_ROOT=$PSI_VIDEO_CLIP_DATASET_ROOT/2264_with_audio
|
| 37 |
+
PSI_DANCE_VIDEO_CLIP_EMBED_ROOT=${PSI_DANCE_VIDEO_CLIP_EMBED_ROOT:-$PSI_DANCE_VIDEO_CLIP_ROOT/embeds}
|
| 38 |
+
PSI_MOTION_EXAMPLE_VIDEO_CLIP_EMBED_ROOT=${PSI_MOTION_EXAMPLE_VIDEO_CLIP_EMBED_ROOT:-$PSI_MOTION_EXAMPLE_VIDEO_CLIP_ROOT/embeds}
|
| 39 |
+
PSI_2264_WITH_AUDIO_VIDEO_CLIP_EMBED_ROOT=${PSI_2264_WITH_AUDIO_VIDEO_CLIP_EMBED_ROOT:-$PSI_2264_WITH_AUDIO_VIDEO_CLIP_ROOT/embeds}
|
| 40 |
+
PSI_MOTION_EXAMPLE_VIDEO_EMBED_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/videogen/motion_example_0506_demo_video_embeds
|
| 41 |
+
PSI_AUDIO_RECORDS_CACHE_DIR=${PSI_AUDIO_RECORDS_CACHE_DIR:-$PSI_2264_WITH_AUDIO_ROOT/.index_cache_multimodal_yth}
|
| 42 |
+
PSI_AUDIO_COMMAND_RECORDS_CACHE_DIR=${PSI_AUDIO_COMMAND_RECORDS_CACHE_DIR:-$PSI_QWEN3_TTS_ROOT/.index_cache_multimodal_yth}
|
| 43 |
+
PSI_DANCE_AUDIO_NPY_ROOT=${PSI_DANCE_AUDIO_NPY_ROOT:-$PSI_DANCE_AUDIO_VIDEO_ROOT/audio_npy_yth}
|
| 44 |
+
PSI_DANCE_AUDIO_NPY_MANIFEST=${PSI_DANCE_AUDIO_NPY_MANIFEST:-$PSI_DANCE_AUDIO_VIDEO_ROOT/metadata_audio_npy.jsonl}
|
| 45 |
+
PSI_2264_AUDIO_NPY_ROOT=${PSI_2264_AUDIO_NPY_ROOT:-$PSI_2264_WITH_AUDIO_ROOT/audio_npy_yth}
|
| 46 |
+
PSI_2264_AUDIO_NPY_MANIFEST=${PSI_2264_AUDIO_NPY_MANIFEST:-$PSI_2264_WITH_AUDIO_ROOT/metadata_audio_npy.jsonl}
|
| 47 |
+
PSI_QWEN3_TTS_AUDIO_NPY_ROOT=${PSI_QWEN3_TTS_AUDIO_NPY_ROOT:-$PSI_QWEN3_TTS_ROOT/audio_npy_yth}
|
| 48 |
+
PSI_QWEN3_TTS_AUDIO_NPY_MANIFEST=${PSI_QWEN3_TTS_AUDIO_NPY_MANIFEST:-$PSI_QWEN3_TTS_ROOT/metadata_audio_npy.jsonl}
|
| 49 |
+
|
| 50 |
+
if [ -f "$PSI_2264_AUDIO_NPY_MANIFEST" ]; then
|
| 51 |
+
PSI_2264_AUDIO_METADATA_JSONL=$PSI_2264_AUDIO_NPY_MANIFEST
|
| 52 |
+
PSI_2264_AUDIO_PATH_KEY=audio_path
|
| 53 |
+
else
|
| 54 |
+
PSI_2264_AUDIO_METADATA_JSONL=metadata_convert.jsonl
|
| 55 |
+
PSI_2264_AUDIO_PATH_KEY=video_path
|
| 56 |
+
fi
|
| 57 |
+
|
| 58 |
+
if [ -f "$PSI_DANCE_AUDIO_NPY_MANIFEST" ]; then
|
| 59 |
+
PSI_DANCE_AUDIO_SOURCE_JSON=$(cat <<JSON
|
| 60 |
+
{
|
| 61 |
+
"name": "psi0_dance_audio",
|
| 62 |
+
"dataset_root": "$PSI_DANCE_AUDIO_VIDEO_ROOT",
|
| 63 |
+
"metadata_jsonl": "$PSI_DANCE_AUDIO_NPY_MANIFEST",
|
| 64 |
+
"motion_dir_key": "converted_path",
|
| 65 |
+
"audio_path_key": "audio_path",
|
| 66 |
+
"records_cache_dir": "$PSI_AUDIO_RECORDS_CACHE_DIR",
|
| 67 |
+
"dataset_name": "seed-audio-g1"
|
| 68 |
+
}
|
| 69 |
+
JSON
|
| 70 |
+
)
|
| 71 |
+
else
|
| 72 |
+
PSI_DANCE_AUDIO_SOURCE_JSON=$(cat <<JSON
|
| 73 |
+
{
|
| 74 |
+
"name": "psi0_dance_audio",
|
| 75 |
+
"root_dir": "$PSI_DANCE_MOTION_ROOT",
|
| 76 |
+
"video_dir": "$PSI_DANCE_AUDIO_VIDEO_ROOT",
|
| 77 |
+
"records_cache_dir": "$PSI_AUDIO_RECORDS_CACHE_DIR",
|
| 78 |
+
"dataset_name": "seed-audio-g1"
|
| 79 |
+
}
|
| 80 |
+
JSON
|
| 81 |
+
)
|
| 82 |
+
fi
|
| 83 |
+
|
| 84 |
+
if [ -z "$PSI_AUDIO_SOURCES_JSON" ]; then
|
| 85 |
+
PSI_AUDIO_SOURCES_JSON=$(compact_json <<JSON
|
| 86 |
+
[
|
| 87 |
+
$PSI_DANCE_AUDIO_SOURCE_JSON,
|
| 88 |
+
{
|
| 89 |
+
"name": "2264_with_audio",
|
| 90 |
+
"dataset_root": "$PSI_2264_WITH_AUDIO_ROOT",
|
| 91 |
+
"metadata_jsonl": "$PSI_2264_AUDIO_METADATA_JSONL",
|
| 92 |
+
"motion_dir_key": "converted_path",
|
| 93 |
+
"audio_path_key": "$PSI_2264_AUDIO_PATH_KEY",
|
| 94 |
+
"records_cache_dir": "$PSI_AUDIO_RECORDS_CACHE_DIR",
|
| 95 |
+
"dataset_name": "2264-with-audio-g1"
|
| 96 |
+
}
|
| 97 |
+
]
|
| 98 |
+
JSON
|
| 99 |
+
)
|
| 100 |
+
fi
|
| 101 |
+
|
| 102 |
+
if [ -f "$PSI_QWEN3_TTS_AUDIO_NPY_MANIFEST" ]; then
|
| 103 |
+
PSI_AUDIO_COMMAND_SOURCE_JSON=$(cat <<JSON
|
| 104 |
+
{
|
| 105 |
+
"name": "qwen3_tts_seed_audio_command",
|
| 106 |
+
"root_dir": "$PSI_SEED_ROOT",
|
| 107 |
+
"dataset_root": "$PSI_QWEN3_TTS_ROOT",
|
| 108 |
+
"metadata_jsonl": "$PSI_QWEN3_TTS_AUDIO_NPY_MANIFEST",
|
| 109 |
+
"motion_dir_key": "filename",
|
| 110 |
+
"audio_path_key": "audio_path",
|
| 111 |
+
"instruction_key": "content_short_description",
|
| 112 |
+
"motion_metadata_csv": "$PSI_SEED_METADATA",
|
| 113 |
+
"motion_metadata_key": "filename",
|
| 114 |
+
"motion_metadata_path_key": "move_g1_path",
|
| 115 |
+
"audio_file_extensions": ".npy",
|
| 116 |
+
"records_cache_dir": "$PSI_AUDIO_COMMAND_RECORDS_CACHE_DIR",
|
| 117 |
+
"audio_task": "command",
|
| 118 |
+
"dataset_name": "seed-audio-command-g1"
|
| 119 |
+
}
|
| 120 |
+
JSON
|
| 121 |
+
)
|
| 122 |
+
else
|
| 123 |
+
PSI_AUDIO_COMMAND_SOURCE_JSON=$(cat <<JSON
|
| 124 |
+
{
|
| 125 |
+
"name": "qwen3_tts_seed_audio_command",
|
| 126 |
+
"root_dir": "$PSI_SEED_ROOT",
|
| 127 |
+
"dataset_root": "$PSI_QWEN3_TTS_ROOT",
|
| 128 |
+
"metadata_jsonl": "metadata.jsonl",
|
| 129 |
+
"motion_dir_key": "filename",
|
| 130 |
+
"audio_path_key": "audio_path",
|
| 131 |
+
"instruction_key": "content_short_description",
|
| 132 |
+
"motion_metadata_csv": "$PSI_SEED_METADATA",
|
| 133 |
+
"motion_metadata_key": "filename",
|
| 134 |
+
"motion_metadata_path_key": "move_g1_path",
|
| 135 |
+
"audio_file_extensions": ".wav",
|
| 136 |
+
"records_cache_dir": "$PSI_AUDIO_COMMAND_RECORDS_CACHE_DIR",
|
| 137 |
+
"audio_task": "command",
|
| 138 |
+
"dataset_name": "seed-audio-command-g1"
|
| 139 |
+
}
|
| 140 |
+
JSON
|
| 141 |
+
)
|
| 142 |
+
fi
|
| 143 |
+
|
| 144 |
+
if [ -z "$PSI_AUDIO_COMMAND_SOURCES_JSON" ]; then
|
| 145 |
+
PSI_AUDIO_COMMAND_SOURCES_JSON=$(compact_json <<JSON
|
| 146 |
+
[
|
| 147 |
+
$PSI_AUDIO_COMMAND_SOURCE_JSON
|
| 148 |
+
]
|
| 149 |
+
JSON
|
| 150 |
+
)
|
| 151 |
+
fi
|
| 152 |
+
|
| 153 |
+
if [ -z "$PSI_VIDEO_SOURCES_JSON" ]; then
|
| 154 |
+
PSI_VIDEO_SOURCES_JSON=$(compact_json <<JSON
|
| 155 |
+
[
|
| 156 |
+
{
|
| 157 |
+
"name": "psi0_dance_gvhmr",
|
| 158 |
+
"root_dir": "$PSI_DANCE_MOTION_ROOT",
|
| 159 |
+
"video_dir": "$PSI_DANCE_VIDEO_ROOT",
|
| 160 |
+
"video_embed_dir": "$PSI_DANCE_VIDEO_EMBED_ROOT",
|
| 161 |
+
"dataset_name": "seed-video-g1"
|
| 162 |
+
},
|
| 163 |
+
{
|
| 164 |
+
"name": "motion_example_0506_demo",
|
| 165 |
+
"dataset_root": "$PSI_VIDEO_DEMO_ROOT",
|
| 166 |
+
"metadata_jsonl": "metadata.jsonl",
|
| 167 |
+
"motion_dir_key": "converted_path",
|
| 168 |
+
"video_path_key": "video_path",
|
| 169 |
+
"instruction_key": "video_caption",
|
| 170 |
+
"video_embed_dir": "$PSI_MOTION_EXAMPLE_VIDEO_EMBED_ROOT",
|
| 171 |
+
"dataset_name": "motion-example-video-g1"
|
| 172 |
+
}
|
| 173 |
+
]
|
| 174 |
+
JSON
|
| 175 |
+
)
|
| 176 |
+
fi
|
| 177 |
+
|
| 178 |
+
if [ -z "$PSI_VIDEO_CLIP_SOURCES_JSON" ]; then
|
| 179 |
+
PSI_VIDEO_CLIP_SOURCES_JSON=$(compact_json <<JSON
|
| 180 |
+
[
|
| 181 |
+
{
|
| 182 |
+
"name": "dance_1723_video_10s_clip50",
|
| 183 |
+
"dataset_root": "$PSI_DANCE_VIDEO_CLIP_ROOT",
|
| 184 |
+
"metadata_jsonl": "metadata.jsonl",
|
| 185 |
+
"motion_dir_key": "converted_path",
|
| 186 |
+
"video_path_key": "video_path",
|
| 187 |
+
"instruction_key": "video_caption",
|
| 188 |
+
"video_embed_dir": "$PSI_DANCE_VIDEO_CLIP_EMBED_ROOT",
|
| 189 |
+
"dataset_name": "dance-video-clip-g1"
|
| 190 |
+
},
|
| 191 |
+
{
|
| 192 |
+
"name": "motion_example_0506_demo_clip50",
|
| 193 |
+
"dataset_root": "$PSI_MOTION_EXAMPLE_VIDEO_CLIP_ROOT",
|
| 194 |
+
"metadata_jsonl": "metadata.jsonl",
|
| 195 |
+
"motion_dir_key": "converted_path",
|
| 196 |
+
"video_path_key": "video_path",
|
| 197 |
+
"instruction_key": "video_caption",
|
| 198 |
+
"video_embed_dir": "$PSI_MOTION_EXAMPLE_VIDEO_CLIP_EMBED_ROOT",
|
| 199 |
+
"dataset_name": "motion-example-video-clip-g1"
|
| 200 |
+
},
|
| 201 |
+
{
|
| 202 |
+
"name": "2264_with_audio_clip50",
|
| 203 |
+
"dataset_root": "$PSI_2264_WITH_AUDIO_VIDEO_CLIP_ROOT",
|
| 204 |
+
"metadata_jsonl": "metadata.jsonl",
|
| 205 |
+
"motion_dir_key": "converted_path",
|
| 206 |
+
"video_path_key": "video_path",
|
| 207 |
+
"instruction_key": "video_caption",
|
| 208 |
+
"video_embed_dir": "$PSI_2264_WITH_AUDIO_VIDEO_CLIP_EMBED_ROOT",
|
| 209 |
+
"dataset_name": "2264-with-audio-video-clip-g1"
|
| 210 |
+
}
|
| 211 |
+
]
|
| 212 |
+
JSON
|
| 213 |
+
)
|
| 214 |
+
fi
|
| 215 |
+
|
| 216 |
+
# 7-task from-scratch mix, excluding legacy VIDEO_TEXT / VIDEO_ONLY.
|
| 217 |
+
# Smaller data sources get larger weights. These use a tempered inverse-size
|
| 218 |
+
# rule, (47M / source_size)^0.25, so rare modalities are upweighted without
|
| 219 |
+
# completely starving text/video_clip in BatchMixtureSampler.
|
| 220 |
+
# text stride1 frame samples: ~47,000,000
|
| 221 |
+
# music audio dance frame samples: 213,731
|
| 222 |
+
# audio_command wav files: 41,760
|
| 223 |
+
# dense video_clip metadata: 4,597,576
|
| 224 |
+
PSI_TEXT_TASK_WEIGHT=1
|
| 225 |
+
PSI_AUDIO_TEXT_TASK_WEIGHT=1
|
| 226 |
+
PSI_AUDIO_ONLY_TASK_WEIGHT=1
|
| 227 |
+
PSI_AUDIO_COMMAND_TEXT_TASK_WEIGHT=1
|
| 228 |
+
PSI_AUDIO_COMMAND_ONLY_TASK_WEIGHT=1
|
| 229 |
+
PSI_VIDEO_TEXT_TASK_WEIGHT=0
|
| 230 |
+
PSI_VIDEO_ONLY_TASK_WEIGHT=0
|
| 231 |
+
PSI_VIDEO_CLIP_TEXT_TASK_WEIGHT=1
|
| 232 |
+
PSI_VIDEO_CLIP_ONLY_TASK_WEIGHT=1
|
| 233 |
+
|
| 234 |
+
PSI_PRETRAIN_BACKBONE_PATH=/cpfs01/VideoGen/public_ckpt/Qwen/Qwen2.5-Omni-3B
|
| 235 |
+
NUM_PAST_FRAMES=0
|
| 236 |
+
STATE_HORIZON=$((NUM_PAST_FRAMES + 1))
|
| 237 |
+
ACTION_CHUNK_SIZE=30
|
| 238 |
+
ACTION_EXEC_HORIZON=20
|
| 239 |
+
PROGRESS_DIM=5
|
| 240 |
+
VIDEO_SAMPLE_FPS=1.0
|
| 241 |
+
VIDEO_CLIP_SAMPLE_FPS=8.0
|
| 242 |
+
VIDEO_SIZE_PIXELS=50176
|
| 243 |
+
VIDEO_DECODER=ffmpeg
|
| 244 |
+
VIDEO_CACHE_SIZE=64
|
| 245 |
+
VIDEO_CLIP_MOTION_CACHE_SIZE=256
|
| 246 |
+
VIDEO_RECORD_BUILD_WORKERS=${VIDEO_RECORD_BUILD_WORKERS:-32}
|
| 247 |
+
VIDEO_RECORD_BUILD_CHUNK_SIZE=${VIDEO_RECORD_BUILD_CHUNK_SIZE:-4096}
|
| 248 |
+
VIDEO_CLIP_TRUST_METADATA=${VIDEO_CLIP_TRUST_METADATA:-1}
|
| 249 |
+
VIDEO_CLIP_RECORDS_CACHE_DIR=${VIDEO_CLIP_RECORDS_CACHE_DIR:-$PSI_VIDEO_CLIP_DATASET_ROOT/.index_cache_multimodal_yth}
|
| 250 |
+
if [ "$VIDEO_CLIP_TRUST_METADATA" = "1" ] || [ "$VIDEO_CLIP_TRUST_METADATA" = "true" ] || [ "$VIDEO_CLIP_TRUST_METADATA" = "True" ]; then
|
| 251 |
+
VIDEO_CLIP_TRUST_METADATA_FLAG="--data.video-clip.trust-metadata"
|
| 252 |
+
else
|
| 253 |
+
VIDEO_CLIP_TRUST_METADATA_FLAG="--data.video-clip.no-trust-metadata"
|
| 254 |
+
fi
|
| 255 |
+
|
| 256 |
+
# DATALOADER_NUM_WORKERS=4
|
| 257 |
+
# VAL_DATALOADER_NUM_WORKERS=2
|
| 258 |
+
|
| 259 |
+
# DATALOADER_NUM_WORKERS=${DATALOADER_NUM_WORKERS:-0}
|
| 260 |
+
# VAL_DATALOADER_NUM_WORKERS=${VAL_DATALOADER_NUM_WORKERS:-0}
|
| 261 |
+
|
| 262 |
+
DATALOADER_NUM_WORKERS=${DATALOADER_NUM_WORKERS:-2}
|
| 263 |
+
VAL_DATALOADER_NUM_WORKERS=${VAL_DATALOADER_NUM_WORKERS:-2}
|
| 264 |
+
|
| 265 |
+
NPROC_PER_NODE=$(echo $CUDA_VISIBLE_DEVICES | tr ',' '\n' | wc -l)
|
| 266 |
+
|
| 267 |
+
echo "Training with $NPROC_PER_NODE GPU(s): $CUDA_VISIBLE_DEVICES"
|
| 268 |
+
echo "Text root: $PSI_SEED_ROOT"
|
| 269 |
+
echo "Text records cache dir: $PSI_TEXT_RECORDS_CACHE_DIR"
|
| 270 |
+
echo "Dance motion root: $PSI_DANCE_MOTION_ROOT"
|
| 271 |
+
echo "Dance audio/video root: $PSI_DANCE_AUDIO_VIDEO_ROOT"
|
| 272 |
+
echo "Dance video root: $PSI_DANCE_VIDEO_ROOT"
|
| 273 |
+
echo "Dance video embed root: $PSI_DANCE_VIDEO_EMBED_ROOT"
|
| 274 |
+
echo "2264 with audio root: $PSI_2264_WITH_AUDIO_ROOT"
|
| 275 |
+
echo "Dance audio npy root: $PSI_DANCE_AUDIO_NPY_ROOT"
|
| 276 |
+
echo "Dance audio manifest: $PSI_DANCE_AUDIO_NPY_MANIFEST"
|
| 277 |
+
echo "2264 audio npy root: $PSI_2264_AUDIO_NPY_ROOT"
|
| 278 |
+
echo "2264 audio manifest: $PSI_2264_AUDIO_METADATA_JSONL"
|
| 279 |
+
echo "Audio records cache dir: $PSI_AUDIO_RECORDS_CACHE_DIR"
|
| 280 |
+
echo "Audio command records cache dir: $PSI_AUDIO_COMMAND_RECORDS_CACHE_DIR"
|
| 281 |
+
echo "Qwen3-TTS acdudio command root: $PSI_QWEN3_TTS_ROOT"
|
| 282 |
+
echo "Qwen3-TTS audio npy root: $PSI_QWEN3_TTS_AUDIO_NPY_ROOT"
|
| 283 |
+
echo "Qwen3-TTS audio manifest: $PSI_QWEN3_TTS_AUDIO_NPY_MANIFEST"
|
| 284 |
+
echo "Audio sources: $PSI_AUDIO_SOURCES_JSON"
|
| 285 |
+
echo "Audio command sources: $PSI_AUDIO_COMMAND_SOURCES_JSON"
|
| 286 |
+
echo "Video sources: $PSI_VIDEO_SOURCES_JSON"
|
| 287 |
+
echo "Video clip sources: $PSI_VIDEO_CLIP_SOURCES_JSON"
|
| 288 |
+
echo "DataLoader workers: train=$DATALOADER_NUM_WORKERS val=$VAL_DATALOADER_NUM_WORKERS"
|
| 289 |
+
echo "Sampler: task_mixture"
|
| 290 |
+
echo "Task weights: text=$PSI_TEXT_TASK_WEIGHT audio_text=$PSI_AUDIO_TEXT_TASK_WEIGHT audio_only=$PSI_AUDIO_ONLY_TASK_WEIGHT audio_command_text=$PSI_AUDIO_COMMAND_TEXT_TASK_WEIGHT audio_command_only=$PSI_AUDIO_COMMAND_ONLY_TASK_WEIGHT video_text=$PSI_VIDEO_TEXT_TASK_WEIGHT video_only=$PSI_VIDEO_ONLY_TASK_WEIGHT video_clip_text=$PSI_VIDEO_CLIP_TEXT_TASK_WEIGHT video_clip_only=$PSI_VIDEO_CLIP_ONLY_TASK_WEIGHT"
|
| 291 |
+
# echo "Resume run dir: $PSI_PRETRAIN_RUN_DIR"
|
| 292 |
+
echo "TextOp 39D action/state enabled"
|
| 293 |
+
echo "Future progress embedding: enabled, dim $PROGRESS_DIM"
|
| 294 |
+
echo "Backbone path: $PSI_PRETRAIN_BACKBONE_PATH"
|
| 295 |
+
echo "Output dir: $PSI_OUTPUT_DIR"
|
| 296 |
+
echo "Video clip trust metadata: $VIDEO_CLIP_TRUST_METADATA"
|
| 297 |
+
echo "Video clip records cache dir: $VIDEO_CLIP_RECORDS_CACHE_DIR"
|
| 298 |
+
echo "Video record build workers: $VIDEO_RECORD_BUILD_WORKERS"
|
| 299 |
+
echo "Video record build chunk size: $VIDEO_RECORD_BUILD_CHUNK_SIZE"
|
| 300 |
+
|
| 301 |
+
args="posttrain_v1_seed_text_audio_video_mix_psi0_config \
|
| 302 |
+
--seed=292285 \
|
| 303 |
+
--exp=posttrain \
|
| 304 |
+
--timestamp=$(date +"%y%m%d%H%M") \
|
| 305 |
+
--train.name=posttrain_v1_trimodal \
|
| 306 |
+
--train.data_parallel=ddp \
|
| 307 |
+
--train.mixed_precision=bf16 \
|
| 308 |
+
--train.train_batch_size=64 \
|
| 309 |
+
--train.val_batch_size=64 \
|
| 310 |
+
--train.num-workers=$DATALOADER_NUM_WORKERS \
|
| 311 |
+
--train.val-num-workers=$VAL_DATALOADER_NUM_WORKERS \
|
| 312 |
+
--train.resume_from_checkpoint=None \
|
| 313 |
+
--train.max_checkpoints_to_keep=1 \
|
| 314 |
+
--train.gradient_accumulation_steps=1 \
|
| 315 |
+
--train.learning_rate=1e-4 \
|
| 316 |
+
--train.max_training_steps=500000 \
|
| 317 |
+
--train.warmup_ratio=None \
|
| 318 |
+
--train.warmup_steps=1000 \
|
| 319 |
+
--train.checkpointing_steps=1000 \
|
| 320 |
+
--train.validation_steps=1000 \
|
| 321 |
+
--train.val_num_batches=20 \
|
| 322 |
+
--train.max_grad_norm=1.0 \
|
| 323 |
+
--train.lr_scheduler_type=cosine_with_min_lr \
|
| 324 |
+
--train.lr_scheduler_kwargs.betas 0.9 0.999 \
|
| 325 |
+
--train.lr_scheduler_kwargs.weight_decay=0 \
|
| 326 |
+
--train.lr_scheduler_kwargs.eps=1e-8 \
|
| 327 |
+
--log.report_to=$PSI_LOG_REPORT_TO \
|
| 328 |
+
--train.output_dir=$PSI_OUTPUT_DIR \
|
| 329 |
+
--data.text.root-dir=$PSI_SEED_ROOT \
|
| 330 |
+
--data.text.metadata-csv=$PSI_SEED_METADATA \
|
| 331 |
+
--data.text.records-cache-dir=$PSI_TEXT_RECORDS_CACHE_DIR \
|
| 332 |
+
--data.text-ratio=$PSI_TEXT_TASK_WEIGHT \
|
| 333 |
+
--data.audio.sources-json=$PSI_AUDIO_SOURCES_JSON \
|
| 334 |
+
--data.audio-ratio=$PSI_AUDIO_TEXT_TASK_WEIGHT \
|
| 335 |
+
--data.audio-only-ratio=$PSI_AUDIO_ONLY_TASK_WEIGHT \
|
| 336 |
+
--data.audio.sample-stride=1 \
|
| 337 |
+
--data.audio.motion-fps=50 \
|
| 338 |
+
--data.audio.audio-sampling-rate=16000 \
|
| 339 |
+
--data.audio.audio-window-seconds=1.0 \
|
| 340 |
+
--data.audio.audio-left-context-seconds=0.0 \
|
| 341 |
+
--data.audio.full-audio \
|
| 342 |
+
--data.audio-command.sources-json=$PSI_AUDIO_COMMAND_SOURCES_JSON \
|
| 343 |
+
--data.audio-command-ratio=$PSI_AUDIO_COMMAND_TEXT_TASK_WEIGHT \
|
| 344 |
+
--data.audio-command-only-ratio=$PSI_AUDIO_COMMAND_ONLY_TASK_WEIGHT \
|
| 345 |
+
--data.audio-command.sample-stride=1 \
|
| 346 |
+
--data.audio-command.motion-fps=50 \
|
| 347 |
+
--data.audio-command.audio-sampling-rate=16000 \
|
| 348 |
+
--data.audio-command.audio-window-seconds=1.0 \
|
| 349 |
+
--data.audio-command.audio-left-context-seconds=0.0 \
|
| 350 |
+
--data.audio-command.full-audio \
|
| 351 |
+
--data.video-ratio=$PSI_VIDEO_TEXT_TASK_WEIGHT \
|
| 352 |
+
--data.video-only-ratio=$PSI_VIDEO_ONLY_TASK_WEIGHT \
|
| 353 |
+
--data.video.sample-stride=1 \
|
| 354 |
+
--data.video.motion-fps=50 \
|
| 355 |
+
--data.video.video-sample-fps=$VIDEO_SAMPLE_FPS \
|
| 356 |
+
--data.video.video-size-pixels=$VIDEO_SIZE_PIXELS \
|
| 357 |
+
--data.video.video-decoder=$VIDEO_DECODER \
|
| 358 |
+
--data.video.video-cache-size=$VIDEO_CACHE_SIZE \
|
| 359 |
+
--data.video.record-build-workers=$VIDEO_RECORD_BUILD_WORKERS \
|
| 360 |
+
--data.video.record-build-chunk-size=$VIDEO_RECORD_BUILD_CHUNK_SIZE \
|
| 361 |
+
--data.video.sources-json=$PSI_VIDEO_SOURCES_JSON \
|
| 362 |
+
--data.video-clip-text-ratio=$PSI_VIDEO_CLIP_TEXT_TASK_WEIGHT \
|
| 363 |
+
--data.video-clip-only-ratio=$PSI_VIDEO_CLIP_ONLY_TASK_WEIGHT \
|
| 364 |
+
--data.video-clip.sources-json=$PSI_VIDEO_CLIP_SOURCES_JSON \
|
| 365 |
+
--data.video-clip.sample-stride=1 \
|
| 366 |
+
--data.video-clip.motion-fps=50 \
|
| 367 |
+
--data.video-clip.video-sample-fps=$VIDEO_CLIP_SAMPLE_FPS \
|
| 368 |
+
--data.video-clip.video-size-pixels=$VIDEO_SIZE_PIXELS \
|
| 369 |
+
--data.video-clip.video-decoder=$VIDEO_DECODER \
|
| 370 |
+
--data.video-clip.video-cache-size=$VIDEO_CACHE_SIZE \
|
| 371 |
+
--data.video-clip.motion-cache-size=$VIDEO_CLIP_MOTION_CACHE_SIZE \
|
| 372 |
+
--data.video-clip.record-build-workers=$VIDEO_RECORD_BUILD_WORKERS \
|
| 373 |
+
--data.video-clip.record-build-chunk-size=$VIDEO_RECORD_BUILD_CHUNK_SIZE \
|
| 374 |
+
$VIDEO_CLIP_TRUST_METADATA_FLAG \
|
| 375 |
+
--data.video-clip.records-cache-dir=$VIDEO_CLIP_RECORDS_CACHE_DIR \
|
| 376 |
+
--data.video-clip.action-exec-horizon=$ACTION_EXEC_HORIZON \
|
| 377 |
+
--data.sampler=task_mixture \
|
| 378 |
+
--data.val-task=text,audio,audio_only,audio_command,audio_command_only,video_clip,video_clip_only \
|
| 379 |
+
--data.transform.repack.num-past-frames=$NUM_PAST_FRAMES \
|
| 380 |
+
--data.transform.repack.textop-actions \
|
| 381 |
+
--data.transform.repack.action-chunk-size=$ACTION_CHUNK_SIZE \
|
| 382 |
+
--data.transform.repack.pad-action-dim=39 \
|
| 383 |
+
--data.transform.repack.pad-state-dim=39 \
|
| 384 |
+
--data.transform.field.action_norm_type=bounds_q99 \
|
| 385 |
+
--data.transform.field.stat-path=assets/stats/seed_all_g1_motion39_textop_stats.json \
|
| 386 |
+
--data.transform.field.normalize-state \
|
| 387 |
+
--data.transform.field.pad-action-dim=39 \
|
| 388 |
+
--data.transform.field.pad-state-dim=39 \
|
| 389 |
+
--data.transform.model.resize.size 240 320 \
|
| 390 |
+
--data.transform.model.center_crop.size 240 320 \
|
| 391 |
+
--data.transform.model.no-img-aug \
|
| 392 |
+
--data.transform.model.text-only \
|
| 393 |
+
--data.transform.model.audio-only \
|
| 394 |
+
--data.transform.model.video-only \
|
| 395 |
+
--data.transform.model.video-size-pixels=$VIDEO_SIZE_PIXELS \
|
| 396 |
+
--model.model_name_or_path=$PSI_PRETRAIN_BACKBONE_PATH \
|
| 397 |
+
--model.noise-scheduler=flow \
|
| 398 |
+
--model.n_conditions=0 \
|
| 399 |
+
--model.action-chunk-size=$ACTION_CHUNK_SIZE \
|
| 400 |
+
--model.action-dim=39 \
|
| 401 |
+
--model.action-exec-horizon=$ACTION_EXEC_HORIZON \
|
| 402 |
+
--model.observation-horizon=$STATE_HORIZON \
|
| 403 |
+
--model.odim=39 \
|
| 404 |
+
--model.view_feature_dim=2048 \
|
| 405 |
+
--model.use-future-progress-embedding \
|
| 406 |
+
--model.future-progress-dim=$PROGRESS_DIM \
|
| 407 |
+
--model.tune-vlm \
|
| 408 |
+
--model.lang-backbone-lr=1e-5 \
|
| 409 |
+
--model.no-use_film \
|
| 410 |
+
--model.no-combined_temb \
|
| 411 |
+
--model.rtc \
|
| 412 |
+
--model.max-delay=11 \
|
| 413 |
+
--train.max-grad-value=1 \
|
| 414 |
+
--train.loss-group-weight.joint-abs=1.0 \
|
| 415 |
+
--train.loss-group-weight.root-delta-xy=3.0 \
|
| 416 |
+
--train.loss-group-weight.root-rot5-delta=10.0 \
|
| 417 |
+
--train.loss-group-weight.root-height=15.0 \
|
| 418 |
+
--train.loss-group-weight.foot-contact=0.1
|
| 419 |
+
"
|
| 420 |
+
torchrun --nproc_per_node=$NPROC_PER_NODE --master_port=29500 scripts/train.py \
|
| 421 |
+
$args "$@"
|
posttrain-v1-psi0-all-7task-tune-vlm-direct-dense.sh
ADDED
|
@@ -0,0 +1,305 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
pkill -f monitor.py
|
| 2 |
+
|
| 3 |
+
cd /cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/Psi0-yth-v21
|
| 4 |
+
export PYTHONPATH=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/Psi0-yth-v21/src
|
| 5 |
+
export CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7
|
| 6 |
+
export PSI_OUTPUT_DIR=${PSI_OUTPUT_DIR:-/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/Psi0-yth-v21/runs}
|
| 7 |
+
export PSI_LOG_REPORT_TO=wandb
|
| 8 |
+
|
| 9 |
+
source /cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/Psi0-yth-v15/.venv-psi/bin/activate
|
| 10 |
+
[ -f .env ] && source .env
|
| 11 |
+
|
| 12 |
+
export WANDB_API_KEY="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"
|
| 13 |
+
|
| 14 |
+
export OMP_NUM_THREADS=8
|
| 15 |
+
export TF_CPP_MIN_LOG_LEVEL=3
|
| 16 |
+
|
| 17 |
+
# Keep JSON sources readable in this script, but pass compact JSON to tyro.
|
| 18 |
+
compact_json() {
|
| 19 |
+
python -c 'import json, sys; print(json.dumps(json.load(sys.stdin), separators=(",", ":")))'
|
| 20 |
+
}
|
| 21 |
+
|
| 22 |
+
PSI_SEED_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/Psi0/dataset/seed_all
|
| 23 |
+
PSI_SEED_METADATA=metadata/seed_metadata_v004.csv
|
| 24 |
+
|
| 25 |
+
PSI_DANCE_MOTION_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/GR00T-WholeBodyControl/gear_sonic_deploy/reference/converted/psi0_dance/gvhmr
|
| 26 |
+
PSI_DANCE_AUDIO_VIDEO_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/dance_1723_video_psi0/dance_1723_video_10s
|
| 27 |
+
PSI_DANCE_VIDEO_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/dance_1723_video_psi0/dance_1723_video_10s
|
| 28 |
+
PSI_DANCE_VIDEO_EMBED_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/dance_1723_video_psi0/dance_1723_video_10s_embeds
|
| 29 |
+
PSI_VIDEO_DEMO_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/videogen/motion_example_0506_demo
|
| 30 |
+
PSI_QWEN3_TTS_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/qwen3-tts
|
| 31 |
+
PSI_VIDEO_CLIP_DATASET_ROOT=${PSI_VIDEO_CLIP_DATASET_ROOT:-/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/videogen/psi0_video_clip_50f_dense_stride1_direct_full}
|
| 32 |
+
PSI_DANCE_VIDEO_CLIP_ROOT=$PSI_VIDEO_CLIP_DATASET_ROOT/dance_1723_video_10s
|
| 33 |
+
PSI_MOTION_EXAMPLE_VIDEO_CLIP_ROOT=$PSI_VIDEO_CLIP_DATASET_ROOT/motion_example_0506_demo
|
| 34 |
+
PSI_DANCE_VIDEO_CLIP_EMBED_ROOT=${PSI_DANCE_VIDEO_CLIP_EMBED_ROOT:-$PSI_DANCE_VIDEO_CLIP_ROOT/embeds}
|
| 35 |
+
PSI_MOTION_EXAMPLE_VIDEO_CLIP_EMBED_ROOT=${PSI_MOTION_EXAMPLE_VIDEO_CLIP_EMBED_ROOT:-$PSI_MOTION_EXAMPLE_VIDEO_CLIP_ROOT/embeds}
|
| 36 |
+
PSI_MOTION_EXAMPLE_VIDEO_EMBED_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/videogen/motion_example_0506_demo_video_embeds
|
| 37 |
+
|
| 38 |
+
if [ -z "$PSI_AUDIO_SOURCES_JSON" ]; then
|
| 39 |
+
PSI_AUDIO_SOURCES_JSON=$(compact_json <<JSON
|
| 40 |
+
[
|
| 41 |
+
{
|
| 42 |
+
"name": "psi0_dance_audio",
|
| 43 |
+
"root_dir": "$PSI_DANCE_MOTION_ROOT",
|
| 44 |
+
"video_dir": "$PSI_DANCE_AUDIO_VIDEO_ROOT",
|
| 45 |
+
"dataset_name": "seed-audio-g1"
|
| 46 |
+
}
|
| 47 |
+
]
|
| 48 |
+
JSON
|
| 49 |
+
)
|
| 50 |
+
fi
|
| 51 |
+
|
| 52 |
+
if [ -z "$PSI_AUDIO_COMMAND_SOURCES_JSON" ]; then
|
| 53 |
+
PSI_AUDIO_COMMAND_SOURCES_JSON=$(compact_json <<JSON
|
| 54 |
+
[
|
| 55 |
+
{
|
| 56 |
+
"name": "qwen3_tts_seed_audio_command",
|
| 57 |
+
"root_dir": "$PSI_SEED_ROOT",
|
| 58 |
+
"dataset_root": "$PSI_QWEN3_TTS_ROOT",
|
| 59 |
+
"metadata_jsonl": "metadata.jsonl",
|
| 60 |
+
"motion_dir_key": "filename",
|
| 61 |
+
"audio_path_key": "audio_path",
|
| 62 |
+
"instruction_key": "content_short_description",
|
| 63 |
+
"motion_metadata_csv": "$PSI_SEED_METADATA",
|
| 64 |
+
"motion_metadata_key": "filename",
|
| 65 |
+
"motion_metadata_path_key": "move_g1_path",
|
| 66 |
+
"audio_file_extensions": ".wav",
|
| 67 |
+
"audio_task": "command",
|
| 68 |
+
"dataset_name": "seed-audio-command-g1"
|
| 69 |
+
}
|
| 70 |
+
]
|
| 71 |
+
JSON
|
| 72 |
+
)
|
| 73 |
+
fi
|
| 74 |
+
|
| 75 |
+
if [ -z "$PSI_VIDEO_SOURCES_JSON" ]; then
|
| 76 |
+
PSI_VIDEO_SOURCES_JSON=$(compact_json <<JSON
|
| 77 |
+
[
|
| 78 |
+
{
|
| 79 |
+
"name": "psi0_dance_gvhmr",
|
| 80 |
+
"root_dir": "$PSI_DANCE_MOTION_ROOT",
|
| 81 |
+
"video_dir": "$PSI_DANCE_VIDEO_ROOT",
|
| 82 |
+
"video_embed_dir": "$PSI_DANCE_VIDEO_EMBED_ROOT",
|
| 83 |
+
"dataset_name": "seed-video-g1"
|
| 84 |
+
},
|
| 85 |
+
{
|
| 86 |
+
"name": "motion_example_0506_demo",
|
| 87 |
+
"dataset_root": "$PSI_VIDEO_DEMO_ROOT",
|
| 88 |
+
"metadata_jsonl": "metadata.jsonl",
|
| 89 |
+
"motion_dir_key": "converted_path",
|
| 90 |
+
"video_path_key": "video_path",
|
| 91 |
+
"instruction_key": "video_caption",
|
| 92 |
+
"video_embed_dir": "$PSI_MOTION_EXAMPLE_VIDEO_EMBED_ROOT",
|
| 93 |
+
"dataset_name": "motion-example-video-g1"
|
| 94 |
+
}
|
| 95 |
+
]
|
| 96 |
+
JSON
|
| 97 |
+
)
|
| 98 |
+
fi
|
| 99 |
+
|
| 100 |
+
if [ -z "$PSI_VIDEO_CLIP_SOURCES_JSON" ]; then
|
| 101 |
+
PSI_VIDEO_CLIP_SOURCES_JSON=$(compact_json <<JSON
|
| 102 |
+
[
|
| 103 |
+
{
|
| 104 |
+
"name": "dance_1723_video_10s_clip50",
|
| 105 |
+
"dataset_root": "$PSI_DANCE_VIDEO_CLIP_ROOT",
|
| 106 |
+
"metadata_jsonl": "metadata.jsonl",
|
| 107 |
+
"motion_dir_key": "converted_path",
|
| 108 |
+
"video_path_key": "video_path",
|
| 109 |
+
"instruction_key": "video_caption",
|
| 110 |
+
"video_embed_dir": "$PSI_DANCE_VIDEO_CLIP_EMBED_ROOT",
|
| 111 |
+
"dataset_name": "dance-video-clip-g1"
|
| 112 |
+
},
|
| 113 |
+
{
|
| 114 |
+
"name": "motion_example_0506_demo_clip50",
|
| 115 |
+
"dataset_root": "$PSI_MOTION_EXAMPLE_VIDEO_CLIP_ROOT",
|
| 116 |
+
"metadata_jsonl": "metadata.jsonl",
|
| 117 |
+
"motion_dir_key": "converted_path",
|
| 118 |
+
"video_path_key": "video_path",
|
| 119 |
+
"instruction_key": "video_caption",
|
| 120 |
+
"video_embed_dir": "$PSI_MOTION_EXAMPLE_VIDEO_CLIP_EMBED_ROOT",
|
| 121 |
+
"dataset_name": "motion-example-video-clip-g1"
|
| 122 |
+
}
|
| 123 |
+
]
|
| 124 |
+
JSON
|
| 125 |
+
)
|
| 126 |
+
fi
|
| 127 |
+
|
| 128 |
+
# 7-task from-scratch mix, excluding legacy VIDEO_TEXT / VIDEO_ONLY.
|
| 129 |
+
# Smaller data sources get larger weights. These use a tempered inverse-size
|
| 130 |
+
# rule, (47M / source_size)^0.25, so rare modalities are upweighted without
|
| 131 |
+
# completely starving text/video_clip in BatchMixtureSampler.
|
| 132 |
+
# text stride1 frame samples: ~47,000,000
|
| 133 |
+
# music audio dance frame samples: 213,731
|
| 134 |
+
# audio_command wav files: 41,760
|
| 135 |
+
# dense video_clip metadata: 4,597,576
|
| 136 |
+
PSI_TEXT_TASK_WEIGHT=1.0
|
| 137 |
+
PSI_AUDIO_TEXT_TASK_WEIGHT=3.8509
|
| 138 |
+
PSI_AUDIO_ONLY_TASK_WEIGHT=3.8509
|
| 139 |
+
PSI_AUDIO_COMMAND_TEXT_TASK_WEIGHT=5.7921
|
| 140 |
+
PSI_AUDIO_COMMAND_ONLY_TASK_WEIGHT=5.7921
|
| 141 |
+
PSI_VIDEO_TEXT_TASK_WEIGHT=0
|
| 142 |
+
PSI_VIDEO_ONLY_TASK_WEIGHT=0
|
| 143 |
+
PSI_VIDEO_CLIP_TEXT_TASK_WEIGHT=1.7881
|
| 144 |
+
PSI_VIDEO_CLIP_ONLY_TASK_WEIGHT=1.7881
|
| 145 |
+
|
| 146 |
+
PSI_PRETRAIN_BACKBONE_PATH=/cpfs01/VideoGen/public_ckpt/Qwen/Qwen2.5-Omni-3B
|
| 147 |
+
NUM_PAST_FRAMES=0
|
| 148 |
+
STATE_HORIZON=$((NUM_PAST_FRAMES + 1))
|
| 149 |
+
ACTION_CHUNK_SIZE=30
|
| 150 |
+
ACTION_EXEC_HORIZON=20
|
| 151 |
+
PROGRESS_DIM=5
|
| 152 |
+
VIDEO_SAMPLE_FPS=1.0
|
| 153 |
+
VIDEO_CLIP_SAMPLE_FPS=8.0
|
| 154 |
+
VIDEO_SIZE_PIXELS=50176
|
| 155 |
+
VIDEO_DECODER=ffmpeg
|
| 156 |
+
VIDEO_CACHE_SIZE=64
|
| 157 |
+
VIDEO_CLIP_MOTION_CACHE_SIZE=256
|
| 158 |
+
VIDEO_CLIP_TRUST_METADATA=${VIDEO_CLIP_TRUST_METADATA:-1}
|
| 159 |
+
VIDEO_CLIP_RECORDS_CACHE_DIR=${VIDEO_CLIP_RECORDS_CACHE_DIR:-$PSI_VIDEO_CLIP_DATASET_ROOT/.index_cache}
|
| 160 |
+
if [ "$VIDEO_CLIP_TRUST_METADATA" = "1" ] || [ "$VIDEO_CLIP_TRUST_METADATA" = "true" ] || [ "$VIDEO_CLIP_TRUST_METADATA" = "True" ]; then
|
| 161 |
+
VIDEO_CLIP_TRUST_METADATA_FLAG="--data.video-clip.trust-metadata"
|
| 162 |
+
else
|
| 163 |
+
VIDEO_CLIP_TRUST_METADATA_FLAG="--data.video-clip.no-trust-metadata"
|
| 164 |
+
fi
|
| 165 |
+
DATALOADER_NUM_WORKERS=4
|
| 166 |
+
VAL_DATALOADER_NUM_WORKERS=2
|
| 167 |
+
|
| 168 |
+
NPROC_PER_NODE=$(echo $CUDA_VISIBLE_DEVICES | tr ',' '\n' | wc -l)
|
| 169 |
+
|
| 170 |
+
echo "Training with $NPROC_PER_NODE GPU(s): $CUDA_VISIBLE_DEVICES"
|
| 171 |
+
echo "Text root: $PSI_SEED_ROOT"
|
| 172 |
+
echo "Dance motion root: $PSI_DANCE_MOTION_ROOT"
|
| 173 |
+
echo "Dance audio/video root: $PSI_DANCE_AUDIO_VIDEO_ROOT"
|
| 174 |
+
echo "Dance video root: $PSI_DANCE_VIDEO_ROOT"
|
| 175 |
+
echo "Dance video embed root: $PSI_DANCE_VIDEO_EMBED_ROOT"
|
| 176 |
+
echo "Qwen3-TTS acdudio command root: $PSI_QWEN3_TTS_ROOT"
|
| 177 |
+
echo "Audio sources: $PSI_AUDIO_SOURCES_JSON"
|
| 178 |
+
echo "Audio command sources: $PSI_AUDIO_COMMAND_SOURCES_JSON"
|
| 179 |
+
echo "Video sources: $PSI_VIDEO_SOURCES_JSON"
|
| 180 |
+
echo "Video clip sources: $PSI_VIDEO_CLIP_SOURCES_JSON"
|
| 181 |
+
echo "Task weights: text=$PSI_TEXT_TASK_WEIGHT audio_text=$PSI_AUDIO_TEXT_TASK_WEIGHT audio_only=$PSI_AUDIO_ONLY_TASK_WEIGHT audio_command_text=$PSI_AUDIO_COMMAND_TEXT_TASK_WEIGHT audio_command_only=$PSI_AUDIO_COMMAND_ONLY_TASK_WEIGHT video_text=$PSI_VIDEO_TEXT_TASK_WEIGHT video_only=$PSI_VIDEO_ONLY_TASK_WEIGHT video_clip_text=$PSI_VIDEO_CLIP_TEXT_TASK_WEIGHT video_clip_only=$PSI_VIDEO_CLIP_ONLY_TASK_WEIGHT"
|
| 182 |
+
# echo "Resume run dir: $PSI_PRETRAIN_RUN_DIR"
|
| 183 |
+
echo "TextOp 39D action/state enabled"
|
| 184 |
+
echo "Future progress embedding: enabled, dim $PROGRESS_DIM"
|
| 185 |
+
echo "Backbone path: $PSI_PRETRAIN_BACKBONE_PATH"
|
| 186 |
+
echo "Output dir: $PSI_OUTPUT_DIR"
|
| 187 |
+
echo "Video clip trust metadata: $VIDEO_CLIP_TRUST_METADATA"
|
| 188 |
+
echo "Video clip records cache dir: $VIDEO_CLIP_RECORDS_CACHE_DIR"
|
| 189 |
+
|
| 190 |
+
args="posttrain_v1_seed_text_audio_video_mix_psi0_config \
|
| 191 |
+
--seed=292285 \
|
| 192 |
+
--exp=posttrain \
|
| 193 |
+
--timestamp=$(date +"%y%m%d%H%M") \
|
| 194 |
+
--train.name=posttrain_v1_trimodal \
|
| 195 |
+
--train.data_parallel=ddp \
|
| 196 |
+
--train.mixed_precision=bf16 \
|
| 197 |
+
--train.train_batch_size=32 \
|
| 198 |
+
--train.val_batch_size=64 \
|
| 199 |
+
--train.num-workers=$DATALOADER_NUM_WORKERS \
|
| 200 |
+
--train.val-num-workers=$VAL_DATALOADER_NUM_WORKERS \
|
| 201 |
+
--train.resume_from_checkpoint=None \
|
| 202 |
+
--train.max_checkpoints_to_keep=1 \
|
| 203 |
+
--train.gradient_accumulation_steps=1 \
|
| 204 |
+
--train.learning_rate=1e-4 \
|
| 205 |
+
--train.max_training_steps=500000 \
|
| 206 |
+
--train.warmup_ratio=None \
|
| 207 |
+
--train.warmup_steps=1000 \
|
| 208 |
+
--train.checkpointing_steps=1000 \
|
| 209 |
+
--train.validation_steps=1000 \
|
| 210 |
+
--train.val_num_batches=20 \
|
| 211 |
+
--train.max_grad_norm=1.0 \
|
| 212 |
+
--train.lr_scheduler_type=cosine_with_min_lr \
|
| 213 |
+
--train.lr_scheduler_kwargs.betas 0.9 0.999 \
|
| 214 |
+
--train.lr_scheduler_kwargs.weight_decay=0 \
|
| 215 |
+
--train.lr_scheduler_kwargs.eps=1e-8 \
|
| 216 |
+
--log.report_to=$PSI_LOG_REPORT_TO \
|
| 217 |
+
--train.output_dir=$PSI_OUTPUT_DIR \
|
| 218 |
+
--data.text.root-dir=$PSI_SEED_ROOT \
|
| 219 |
+
--data.text.metadata-csv=$PSI_SEED_METADATA \
|
| 220 |
+
--data.text-ratio=$PSI_TEXT_TASK_WEIGHT \
|
| 221 |
+
--data.audio.sources-json=$PSI_AUDIO_SOURCES_JSON \
|
| 222 |
+
--data.audio-ratio=$PSI_AUDIO_TEXT_TASK_WEIGHT \
|
| 223 |
+
--data.audio-only-ratio=$PSI_AUDIO_ONLY_TASK_WEIGHT \
|
| 224 |
+
--data.audio.sample-stride=1 \
|
| 225 |
+
--data.audio.motion-fps=50 \
|
| 226 |
+
--data.audio.audio-sampling-rate=16000 \
|
| 227 |
+
--data.audio.audio-window-seconds=1.0 \
|
| 228 |
+
--data.audio.audio-left-context-seconds=0.0 \
|
| 229 |
+
--data.audio.full-audio \
|
| 230 |
+
--data.audio-command.sources-json=$PSI_AUDIO_COMMAND_SOURCES_JSON \
|
| 231 |
+
--data.audio-command-ratio=$PSI_AUDIO_COMMAND_TEXT_TASK_WEIGHT \
|
| 232 |
+
--data.audio-command-only-ratio=$PSI_AUDIO_COMMAND_ONLY_TASK_WEIGHT \
|
| 233 |
+
--data.audio-command.sample-stride=1 \
|
| 234 |
+
--data.audio-command.motion-fps=50 \
|
| 235 |
+
--data.audio-command.audio-sampling-rate=16000 \
|
| 236 |
+
--data.audio-command.audio-window-seconds=1.0 \
|
| 237 |
+
--data.audio-command.audio-left-context-seconds=0.0 \
|
| 238 |
+
--data.audio-command.full-audio \
|
| 239 |
+
--data.video-ratio=$PSI_VIDEO_TEXT_TASK_WEIGHT \
|
| 240 |
+
--data.video-only-ratio=$PSI_VIDEO_ONLY_TASK_WEIGHT \
|
| 241 |
+
--data.video.sample-stride=1 \
|
| 242 |
+
--data.video.motion-fps=50 \
|
| 243 |
+
--data.video.video-sample-fps=$VIDEO_SAMPLE_FPS \
|
| 244 |
+
--data.video.video-size-pixels=$VIDEO_SIZE_PIXELS \
|
| 245 |
+
--data.video.video-decoder=$VIDEO_DECODER \
|
| 246 |
+
--data.video.video-cache-size=$VIDEO_CACHE_SIZE \
|
| 247 |
+
--data.video.sources-json=$PSI_VIDEO_SOURCES_JSON \
|
| 248 |
+
--data.video-clip-text-ratio=$PSI_VIDEO_CLIP_TEXT_TASK_WEIGHT \
|
| 249 |
+
--data.video-clip-only-ratio=$PSI_VIDEO_CLIP_ONLY_TASK_WEIGHT \
|
| 250 |
+
--data.video-clip.sources-json=$PSI_VIDEO_CLIP_SOURCES_JSON \
|
| 251 |
+
--data.video-clip.sample-stride=1 \
|
| 252 |
+
--data.video-clip.motion-fps=50 \
|
| 253 |
+
--data.video-clip.video-sample-fps=$VIDEO_CLIP_SAMPLE_FPS \
|
| 254 |
+
--data.video-clip.video-size-pixels=$VIDEO_SIZE_PIXELS \
|
| 255 |
+
--data.video-clip.video-decoder=$VIDEO_DECODER \
|
| 256 |
+
--data.video-clip.video-cache-size=$VIDEO_CACHE_SIZE \
|
| 257 |
+
--data.video-clip.motion-cache-size=$VIDEO_CLIP_MOTION_CACHE_SIZE \
|
| 258 |
+
$VIDEO_CLIP_TRUST_METADATA_FLAG \
|
| 259 |
+
--data.video-clip.records-cache-dir=$VIDEO_CLIP_RECORDS_CACHE_DIR \
|
| 260 |
+
--data.video-clip.action-exec-horizon=$ACTION_EXEC_HORIZON \
|
| 261 |
+
--data.sampler=batch_mixture \
|
| 262 |
+
--data.val-task=text,audio,audio_only,audio_command,audio_command_only,video_clip,video_clip_only \
|
| 263 |
+
--data.transform.repack.num-past-frames=$NUM_PAST_FRAMES \
|
| 264 |
+
--data.transform.repack.textop-actions \
|
| 265 |
+
--data.transform.repack.action-chunk-size=$ACTION_CHUNK_SIZE \
|
| 266 |
+
--data.transform.repack.pad-action-dim=39 \
|
| 267 |
+
--data.transform.repack.pad-state-dim=39 \
|
| 268 |
+
--data.transform.field.action_norm_type=bounds_q99 \
|
| 269 |
+
--data.transform.field.stat-path=assets/stats/seed_all_g1_motion39_textop_stats.json \
|
| 270 |
+
--data.transform.field.normalize-state \
|
| 271 |
+
--data.transform.field.pad-action-dim=39 \
|
| 272 |
+
--data.transform.field.pad-state-dim=39 \
|
| 273 |
+
--data.transform.model.resize.size 240 320 \
|
| 274 |
+
--data.transform.model.center_crop.size 240 320 \
|
| 275 |
+
--data.transform.model.no-img-aug \
|
| 276 |
+
--data.transform.model.text-only \
|
| 277 |
+
--data.transform.model.audio-only \
|
| 278 |
+
--data.transform.model.video-only \
|
| 279 |
+
--data.transform.model.video-size-pixels=$VIDEO_SIZE_PIXELS \
|
| 280 |
+
--model.model_name_or_path=$PSI_PRETRAIN_BACKBONE_PATH \
|
| 281 |
+
--model.noise-scheduler=flow \
|
| 282 |
+
--model.n_conditions=0 \
|
| 283 |
+
--model.action-chunk-size=$ACTION_CHUNK_SIZE \
|
| 284 |
+
--model.action-dim=39 \
|
| 285 |
+
--model.action-exec-horizon=$ACTION_EXEC_HORIZON \
|
| 286 |
+
--model.observation-horizon=$STATE_HORIZON \
|
| 287 |
+
--model.odim=39 \
|
| 288 |
+
--model.view_feature_dim=2048 \
|
| 289 |
+
--model.use-future-progress-embedding \
|
| 290 |
+
--model.future-progress-dim=$PROGRESS_DIM \
|
| 291 |
+
--model.tune-vlm \
|
| 292 |
+
--model.lang-backbone-lr=1e-5 \
|
| 293 |
+
--model.no-use_film \
|
| 294 |
+
--model.no-combined_temb \
|
| 295 |
+
--model.rtc \
|
| 296 |
+
--model.max-delay=11 \
|
| 297 |
+
--train.max-grad-value=1 \
|
| 298 |
+
--train.loss-group-weight.joint-abs=1.0 \
|
| 299 |
+
--train.loss-group-weight.root-delta-xy=3.0 \
|
| 300 |
+
--train.loss-group-weight.root-rot5-delta=10.0 \
|
| 301 |
+
--train.loss-group-weight.root-height=15.0 \
|
| 302 |
+
--train.loss-group-weight.foot-contact=0.1
|
| 303 |
+
"
|
| 304 |
+
torchrun --nproc_per_node=$NPROC_PER_NODE --master_port=29500 scripts/train.py \
|
| 305 |
+
$args "$@"
|
posttrain-v1-psi0-all-text-tune-vlm.sh
ADDED
|
@@ -0,0 +1,282 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
pkill -f monitor.py
|
| 2 |
+
|
| 3 |
+
cd /cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/Psi0-yth-v18
|
| 4 |
+
export PYTHONPATH=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/Psi0-yth-v18/src
|
| 5 |
+
export CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7
|
| 6 |
+
export PSI_OUTPUT_DIR=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/Psi0-yth-v18/runs
|
| 7 |
+
export PSI_LOG_REPORT_TO=wandb
|
| 8 |
+
|
| 9 |
+
source .venv-psi/bin/activate
|
| 10 |
+
[ -f .env ] && source .env
|
| 11 |
+
|
| 12 |
+
export WANDB_API_KEY="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"
|
| 13 |
+
|
| 14 |
+
export OMP_NUM_THREADS=8
|
| 15 |
+
export TF_CPP_MIN_LOG_LEVEL=3
|
| 16 |
+
|
| 17 |
+
# Keep JSON sources readable in this script, but pass compact JSON to tyro.
|
| 18 |
+
compact_json() {
|
| 19 |
+
python -c 'import json, sys; print(json.dumps(json.load(sys.stdin), separators=(",", ":")))'
|
| 20 |
+
}
|
| 21 |
+
|
| 22 |
+
PSI_SEED_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/Psi0/dataset/seed_all
|
| 23 |
+
PSI_SEED_METADATA=metadata/seed_metadata_v004.csv
|
| 24 |
+
|
| 25 |
+
PSI_DANCE_MOTION_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/GR00T-WholeBodyControl/gear_sonic_deploy/reference/converted/psi0_dance/gvhmr
|
| 26 |
+
PSI_DANCE_AUDIO_VIDEO_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/dance_1723_video_psi0/dance_1723_video_10s
|
| 27 |
+
PSI_DANCE_VIDEO_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/dance_1723_video_psi0/dance_1723_video_10s
|
| 28 |
+
PSI_DANCE_VIDEO_EMBED_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/dance_1723_video_psi0/dance_1723_video_10s_embeds
|
| 29 |
+
PSI_VIDEO_DEMO_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/videogen/motion_example_0506_demo
|
| 30 |
+
PSI_QWEN3_TTS_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/qwen3-tts
|
| 31 |
+
PSI_VIDEO_CLIP_DATASET_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/videogen/psi0_video_clip_50f_rand8_full
|
| 32 |
+
PSI_DANCE_VIDEO_CLIP_ROOT=$PSI_VIDEO_CLIP_DATASET_ROOT/dance_1723_video_10s
|
| 33 |
+
PSI_MOTION_EXAMPLE_VIDEO_CLIP_ROOT=$PSI_VIDEO_CLIP_DATASET_ROOT/motion_example_0506_demo
|
| 34 |
+
PSI_MOTION_EXAMPLE_VIDEO_EMBED_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/videogen/motion_example_0506_demo_video_embeds
|
| 35 |
+
|
| 36 |
+
if [ -z "$PSI_AUDIO_SOURCES_JSON" ]; then
|
| 37 |
+
PSI_AUDIO_SOURCES_JSON=$(compact_json <<JSON
|
| 38 |
+
[
|
| 39 |
+
{
|
| 40 |
+
"name": "psi0_dance_audio",
|
| 41 |
+
"root_dir": "$PSI_DANCE_MOTION_ROOT",
|
| 42 |
+
"video_dir": "$PSI_DANCE_AUDIO_VIDEO_ROOT",
|
| 43 |
+
"dataset_name": "seed-audio-g1"
|
| 44 |
+
}
|
| 45 |
+
]
|
| 46 |
+
JSON
|
| 47 |
+
)
|
| 48 |
+
fi
|
| 49 |
+
|
| 50 |
+
if [ -z "$PSI_AUDIO_COMMAND_SOURCES_JSON" ]; then
|
| 51 |
+
PSI_AUDIO_COMMAND_SOURCES_JSON=$(compact_json <<JSON
|
| 52 |
+
[
|
| 53 |
+
{
|
| 54 |
+
"name": "qwen3_tts_seed_audio_command",
|
| 55 |
+
"root_dir": "$PSI_SEED_ROOT",
|
| 56 |
+
"dataset_root": "$PSI_QWEN3_TTS_ROOT",
|
| 57 |
+
"metadata_jsonl": "metadata.jsonl",
|
| 58 |
+
"motion_dir_key": "filename",
|
| 59 |
+
"audio_path_key": "audio_path",
|
| 60 |
+
"instruction_key": "content_short_description",
|
| 61 |
+
"motion_metadata_csv": "$PSI_SEED_METADATA",
|
| 62 |
+
"motion_metadata_key": "filename",
|
| 63 |
+
"motion_metadata_path_key": "move_g1_path",
|
| 64 |
+
"audio_file_extensions": ".wav",
|
| 65 |
+
"audio_task": "command",
|
| 66 |
+
"dataset_name": "seed-audio-command-g1"
|
| 67 |
+
}
|
| 68 |
+
]
|
| 69 |
+
JSON
|
| 70 |
+
)
|
| 71 |
+
fi
|
| 72 |
+
|
| 73 |
+
if [ -z "$PSI_VIDEO_SOURCES_JSON" ]; then
|
| 74 |
+
PSI_VIDEO_SOURCES_JSON=$(compact_json <<JSON
|
| 75 |
+
[
|
| 76 |
+
{
|
| 77 |
+
"name": "psi0_dance_gvhmr",
|
| 78 |
+
"root_dir": "$PSI_DANCE_MOTION_ROOT",
|
| 79 |
+
"video_dir": "$PSI_DANCE_VIDEO_ROOT",
|
| 80 |
+
"video_embed_dir": "$PSI_DANCE_VIDEO_EMBED_ROOT",
|
| 81 |
+
"dataset_name": "seed-video-g1"
|
| 82 |
+
},
|
| 83 |
+
{
|
| 84 |
+
"name": "motion_example_0506_demo",
|
| 85 |
+
"dataset_root": "$PSI_VIDEO_DEMO_ROOT",
|
| 86 |
+
"metadata_jsonl": "metadata.jsonl",
|
| 87 |
+
"motion_dir_key": "converted_path",
|
| 88 |
+
"video_path_key": "video_path",
|
| 89 |
+
"instruction_key": "video_caption",
|
| 90 |
+
"video_embed_dir": "$PSI_MOTION_EXAMPLE_VIDEO_EMBED_ROOT",
|
| 91 |
+
"dataset_name": "motion-example-video-g1"
|
| 92 |
+
}
|
| 93 |
+
]
|
| 94 |
+
JSON
|
| 95 |
+
)
|
| 96 |
+
fi
|
| 97 |
+
|
| 98 |
+
if [ -z "$PSI_VIDEO_CLIP_SOURCES_JSON" ]; then
|
| 99 |
+
PSI_VIDEO_CLIP_SOURCES_JSON=$(compact_json <<JSON
|
| 100 |
+
[
|
| 101 |
+
{
|
| 102 |
+
"name": "dance_1723_video_10s_clip50",
|
| 103 |
+
"dataset_root": "$PSI_DANCE_VIDEO_CLIP_ROOT",
|
| 104 |
+
"metadata_jsonl": "metadata.jsonl",
|
| 105 |
+
"motion_dir_key": "converted_path",
|
| 106 |
+
"video_path_key": "video_path",
|
| 107 |
+
"instruction_key": "video_caption",
|
| 108 |
+
"video_embed_dir": "$PSI_DANCE_VIDEO_CLIP_ROOT/embeds",
|
| 109 |
+
"dataset_name": "dance-video-clip-g1"
|
| 110 |
+
},
|
| 111 |
+
{
|
| 112 |
+
"name": "motion_example_0506_demo_clip50",
|
| 113 |
+
"dataset_root": "$PSI_MOTION_EXAMPLE_VIDEO_CLIP_ROOT",
|
| 114 |
+
"metadata_jsonl": "metadata.jsonl",
|
| 115 |
+
"motion_dir_key": "converted_path",
|
| 116 |
+
"video_path_key": "video_path",
|
| 117 |
+
"instruction_key": "video_caption",
|
| 118 |
+
"video_embed_dir": "$PSI_MOTION_EXAMPLE_VIDEO_CLIP_ROOT/embeds",
|
| 119 |
+
"dataset_name": "motion-example-video-clip-g1"
|
| 120 |
+
}
|
| 121 |
+
]
|
| 122 |
+
JSON
|
| 123 |
+
)
|
| 124 |
+
fi
|
| 125 |
+
|
| 126 |
+
PSI_TEXT_TASK_WEIGHT=1
|
| 127 |
+
PSI_AUDIO_TEXT_TASK_WEIGHT=0
|
| 128 |
+
PSI_AUDIO_ONLY_TASK_WEIGHT=0
|
| 129 |
+
PSI_AUDIO_COMMAND_TEXT_TASK_WEIGHT=0
|
| 130 |
+
PSI_AUDIO_COMMAND_ONLY_TASK_WEIGHT=0
|
| 131 |
+
PSI_VIDEO_TEXT_TASK_WEIGHT=0
|
| 132 |
+
PSI_VIDEO_ONLY_TASK_WEIGHT=0
|
| 133 |
+
PSI_VIDEO_CLIP_TEXT_TASK_WEIGHT=0
|
| 134 |
+
PSI_VIDEO_CLIP_ONLY_TASK_WEIGHT=0
|
| 135 |
+
|
| 136 |
+
PSI_PRETRAIN_BACKBONE_PATH=/cpfs01/VideoGen/public_ckpt/Qwen/Qwen2.5-Omni-3B
|
| 137 |
+
NUM_PAST_FRAMES=0
|
| 138 |
+
STATE_HORIZON=$((NUM_PAST_FRAMES + 1))
|
| 139 |
+
ACTION_CHUNK_SIZE=30
|
| 140 |
+
ACTION_EXEC_HORIZON=20
|
| 141 |
+
PROGRESS_DIM=5
|
| 142 |
+
VIDEO_SAMPLE_FPS=1.0
|
| 143 |
+
VIDEO_CLIP_SAMPLE_FPS=8.0
|
| 144 |
+
VIDEO_SIZE_PIXELS=50176
|
| 145 |
+
VIDEO_DECODER=ffmpeg
|
| 146 |
+
VIDEO_CACHE_SIZE=64
|
| 147 |
+
DATALOADER_NUM_WORKERS=4
|
| 148 |
+
VAL_DATALOADER_NUM_WORKERS=2
|
| 149 |
+
|
| 150 |
+
NPROC_PER_NODE=$(echo $CUDA_VISIBLE_DEVICES | tr ',' '\n' | wc -l)
|
| 151 |
+
|
| 152 |
+
echo "Training with $NPROC_PER_NODE GPU(s): $CUDA_VISIBLE_DEVICES"
|
| 153 |
+
echo "Text root: $PSI_SEED_ROOT"
|
| 154 |
+
echo "Dance motion root: $PSI_DANCE_MOTION_ROOT"
|
| 155 |
+
echo "Dance audio/video root: $PSI_DANCE_AUDIO_VIDEO_ROOT"
|
| 156 |
+
echo "Dance video root: $PSI_DANCE_VIDEO_ROOT"
|
| 157 |
+
echo "Dance video embed root: $PSI_DANCE_VIDEO_EMBED_ROOT"
|
| 158 |
+
echo "Qwen3-TTS acdudio command root: $PSI_QWEN3_TTS_ROOT"
|
| 159 |
+
echo "Audio sources: $PSI_AUDIO_SOURCES_JSON"
|
| 160 |
+
echo "Audio command sources: $PSI_AUDIO_COMMAND_SOURCES_JSON"
|
| 161 |
+
echo "Video sources: $PSI_VIDEO_SOURCES_JSON"
|
| 162 |
+
echo "Video clip sources: $PSI_VIDEO_CLIP_SOURCES_JSON"
|
| 163 |
+
echo "Task weights: text=$PSI_TEXT_TASK_WEIGHT audio_text=$PSI_AUDIO_TEXT_TASK_WEIGHT audio_only=$PSI_AUDIO_ONLY_TASK_WEIGHT audio_command_text=$PSI_AUDIO_COMMAND_TEXT_TASK_WEIGHT audio_command_only=$PSI_AUDIO_COMMAND_ONLY_TASK_WEIGHT video_text=$PSI_VIDEO_TEXT_TASK_WEIGHT video_only=$PSI_VIDEO_ONLY_TASK_WEIGHT video_clip_text=$PSI_VIDEO_CLIP_TEXT_TASK_WEIGHT video_clip_only=$PSI_VIDEO_CLIP_ONLY_TASK_WEIGHT"
|
| 164 |
+
# echo "Resume run dir: $PSI_PRETRAIN_RUN_DIR"
|
| 165 |
+
echo "TextOp 39D action/state enabled"
|
| 166 |
+
echo "Future progress embedding: enabled, dim $PROGRESS_DIM"
|
| 167 |
+
echo "Backbone path: $PSI_PRETRAIN_BACKBONE_PATH"
|
| 168 |
+
echo "Output dir: $PSI_OUTPUT_DIR"
|
| 169 |
+
|
| 170 |
+
args="posttrain_v1_seed_text_audio_video_mix_psi0_config \
|
| 171 |
+
--seed=292285 \
|
| 172 |
+
--exp=posttrain \
|
| 173 |
+
--timestamp=$(date +"%y%m%d%H%M") \
|
| 174 |
+
--train.name=posttrain_v1_trimodal \
|
| 175 |
+
--train.data_parallel=ddp \
|
| 176 |
+
--train.mixed_precision=bf16 \
|
| 177 |
+
--train.train_batch_size=32 \
|
| 178 |
+
--train.val_batch_size=64 \
|
| 179 |
+
--train.num-workers=$DATALOADER_NUM_WORKERS \
|
| 180 |
+
--train.val-num-workers=$VAL_DATALOADER_NUM_WORKERS \
|
| 181 |
+
--train.resume_from_checkpoint=None \
|
| 182 |
+
--train.max_checkpoints_to_keep=1 \
|
| 183 |
+
--train.gradient_accumulation_steps=1 \
|
| 184 |
+
--train.learning_rate=1e-4 \
|
| 185 |
+
--train.max_training_steps=500000 \
|
| 186 |
+
--train.warmup_ratio=None \
|
| 187 |
+
--train.warmup_steps=1000 \
|
| 188 |
+
--train.checkpointing_steps=1000 \
|
| 189 |
+
--train.validation_steps=1000 \
|
| 190 |
+
--train.val_num_batches=20 \
|
| 191 |
+
--train.max_grad_norm=1.0 \
|
| 192 |
+
--train.lr_scheduler_type=cosine_with_min_lr \
|
| 193 |
+
--train.lr_scheduler_kwargs.betas 0.9 0.999 \
|
| 194 |
+
--train.lr_scheduler_kwargs.weight_decay=0 \
|
| 195 |
+
--train.lr_scheduler_kwargs.eps=1e-8 \
|
| 196 |
+
--log.report_to=$PSI_LOG_REPORT_TO \
|
| 197 |
+
--train.output_dir=$PSI_OUTPUT_DIR \
|
| 198 |
+
--data.text.root-dir=$PSI_SEED_ROOT \
|
| 199 |
+
--data.text.metadata-csv=$PSI_SEED_METADATA \
|
| 200 |
+
--data.text-ratio=$PSI_TEXT_TASK_WEIGHT \
|
| 201 |
+
--data.audio.sources-json=$PSI_AUDIO_SOURCES_JSON \
|
| 202 |
+
--data.audio-ratio=$PSI_AUDIO_TEXT_TASK_WEIGHT \
|
| 203 |
+
--data.audio-only-ratio=$PSI_AUDIO_ONLY_TASK_WEIGHT \
|
| 204 |
+
--data.audio.sample-stride=1 \
|
| 205 |
+
--data.audio.motion-fps=50 \
|
| 206 |
+
--data.audio.audio-sampling-rate=16000 \
|
| 207 |
+
--data.audio.audio-window-seconds=1.0 \
|
| 208 |
+
--data.audio.audio-left-context-seconds=0.0 \
|
| 209 |
+
--data.audio.full-audio \
|
| 210 |
+
--data.audio-command.sources-json=$PSI_AUDIO_COMMAND_SOURCES_JSON \
|
| 211 |
+
--data.audio-command-ratio=$PSI_AUDIO_COMMAND_TEXT_TASK_WEIGHT \
|
| 212 |
+
--data.audio-command-only-ratio=$PSI_AUDIO_COMMAND_ONLY_TASK_WEIGHT \
|
| 213 |
+
--data.audio-command.sample-stride=1 \
|
| 214 |
+
--data.audio-command.motion-fps=50 \
|
| 215 |
+
--data.audio-command.audio-sampling-rate=16000 \
|
| 216 |
+
--data.audio-command.audio-window-seconds=1.0 \
|
| 217 |
+
--data.audio-command.audio-left-context-seconds=0.0 \
|
| 218 |
+
--data.audio-command.full-audio \
|
| 219 |
+
--data.video-ratio=$PSI_VIDEO_TEXT_TASK_WEIGHT \
|
| 220 |
+
--data.video-only-ratio=$PSI_VIDEO_ONLY_TASK_WEIGHT \
|
| 221 |
+
--data.video.sample-stride=1 \
|
| 222 |
+
--data.video.motion-fps=50 \
|
| 223 |
+
--data.video.video-sample-fps=$VIDEO_SAMPLE_FPS \
|
| 224 |
+
--data.video.video-size-pixels=$VIDEO_SIZE_PIXELS \
|
| 225 |
+
--data.video.video-decoder=$VIDEO_DECODER \
|
| 226 |
+
--data.video.video-cache-size=$VIDEO_CACHE_SIZE \
|
| 227 |
+
--data.video.sources-json=$PSI_VIDEO_SOURCES_JSON \
|
| 228 |
+
--data.video-clip-text-ratio=$PSI_VIDEO_CLIP_TEXT_TASK_WEIGHT \
|
| 229 |
+
--data.video-clip-only-ratio=$PSI_VIDEO_CLIP_ONLY_TASK_WEIGHT \
|
| 230 |
+
--data.video-clip.sources-json=$PSI_VIDEO_CLIP_SOURCES_JSON \
|
| 231 |
+
--data.video-clip.sample-stride=1 \
|
| 232 |
+
--data.video-clip.motion-fps=50 \
|
| 233 |
+
--data.video-clip.video-sample-fps=$VIDEO_CLIP_SAMPLE_FPS \
|
| 234 |
+
--data.video-clip.video-size-pixels=$VIDEO_SIZE_PIXELS \
|
| 235 |
+
--data.video-clip.video-decoder=$VIDEO_DECODER \
|
| 236 |
+
--data.video-clip.video-cache-size=$VIDEO_CACHE_SIZE \
|
| 237 |
+
--data.video-clip.action-exec-horizon=$ACTION_EXEC_HORIZON \
|
| 238 |
+
--data.sampler=batch_mixture \
|
| 239 |
+
--data.val-task=text \
|
| 240 |
+
--data.transform.repack.num-past-frames=$NUM_PAST_FRAMES \
|
| 241 |
+
--data.transform.repack.textop-actions \
|
| 242 |
+
--data.transform.repack.action-chunk-size=$ACTION_CHUNK_SIZE \
|
| 243 |
+
--data.transform.repack.pad-action-dim=39 \
|
| 244 |
+
--data.transform.repack.pad-state-dim=39 \
|
| 245 |
+
--data.transform.field.action_norm_type=bounds_q99 \
|
| 246 |
+
--data.transform.field.stat-path=assets/stats/seed_all_g1_motion39_textop_stats.json \
|
| 247 |
+
--data.transform.field.normalize-state \
|
| 248 |
+
--data.transform.field.pad-action-dim=39 \
|
| 249 |
+
--data.transform.field.pad-state-dim=39 \
|
| 250 |
+
--data.transform.model.resize.size 240 320 \
|
| 251 |
+
--data.transform.model.center_crop.size 240 320 \
|
| 252 |
+
--data.transform.model.no-img-aug \
|
| 253 |
+
--data.transform.model.text-only \
|
| 254 |
+
--data.transform.model.audio-only \
|
| 255 |
+
--data.transform.model.video-only \
|
| 256 |
+
--data.transform.model.video-size-pixels=$VIDEO_SIZE_PIXELS \
|
| 257 |
+
--model.model_name_or_path=$PSI_PRETRAIN_BACKBONE_PATH \
|
| 258 |
+
--model.noise-scheduler=flow \
|
| 259 |
+
--model.n_conditions=0 \
|
| 260 |
+
--model.action-chunk-size=$ACTION_CHUNK_SIZE \
|
| 261 |
+
--model.action-dim=39 \
|
| 262 |
+
--model.action-exec-horizon=$ACTION_EXEC_HORIZON \
|
| 263 |
+
--model.observation-horizon=$STATE_HORIZON \
|
| 264 |
+
--model.odim=39 \
|
| 265 |
+
--model.view_feature_dim=2048 \
|
| 266 |
+
--model.use-future-progress-embedding \
|
| 267 |
+
--model.future-progress-dim=$PROGRESS_DIM \
|
| 268 |
+
--model.tune-vlm \
|
| 269 |
+
--model.lang-backbone-lr=1e-5 \
|
| 270 |
+
--model.no-use_film \
|
| 271 |
+
--model.no-combined_temb \
|
| 272 |
+
--model.rtc \
|
| 273 |
+
--model.max-delay=11 \
|
| 274 |
+
--train.max-grad-value=1 \
|
| 275 |
+
--train.loss-group-weight.joint-abs=1.0 \
|
| 276 |
+
--train.loss-group-weight.root-delta-xy=3.0 \
|
| 277 |
+
--train.loss-group-weight.root-rot5-delta=10.0 \
|
| 278 |
+
--train.loss-group-weight.root-height=15.0 \
|
| 279 |
+
--train.loss-group-weight.foot-contact=0.1
|
| 280 |
+
"
|
| 281 |
+
torchrun --nproc_per_node=$NPROC_PER_NODE --master_port=29500 scripts/train.py \
|
| 282 |
+
$args "$@"
|
posttrain-v1-psi0-all-tune-vlm-direct-dense.sh
ADDED
|
@@ -0,0 +1,297 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
pkill -f monitor.py
|
| 2 |
+
|
| 3 |
+
cd /cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/Psi0-yth-v18
|
| 4 |
+
export PYTHONPATH=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/Psi0-yth-v18/src
|
| 5 |
+
export CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7
|
| 6 |
+
export PSI_OUTPUT_DIR=${PSI_OUTPUT_DIR:-/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/Psi0-yth-v18/runs}
|
| 7 |
+
export PSI_LOG_REPORT_TO=wandb
|
| 8 |
+
|
| 9 |
+
source /cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/Psi0-yth-v15/.venv-psi/bin/activate
|
| 10 |
+
[ -f .env ] && source .env
|
| 11 |
+
|
| 12 |
+
export WANDB_API_KEY="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"
|
| 13 |
+
|
| 14 |
+
export OMP_NUM_THREADS=8
|
| 15 |
+
export TF_CPP_MIN_LOG_LEVEL=3
|
| 16 |
+
|
| 17 |
+
# Keep JSON sources readable in this script, but pass compact JSON to tyro.
|
| 18 |
+
compact_json() {
|
| 19 |
+
python -c 'import json, sys; print(json.dumps(json.load(sys.stdin), separators=(",", ":")))'
|
| 20 |
+
}
|
| 21 |
+
|
| 22 |
+
PSI_SEED_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/Psi0/dataset/seed_all
|
| 23 |
+
PSI_SEED_METADATA=metadata/seed_metadata_v004.csv
|
| 24 |
+
|
| 25 |
+
PSI_DANCE_MOTION_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/GR00T-WholeBodyControl/gear_sonic_deploy/reference/converted/psi0_dance/gvhmr
|
| 26 |
+
PSI_DANCE_AUDIO_VIDEO_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/dance_1723_video_psi0/dance_1723_video_10s
|
| 27 |
+
PSI_DANCE_VIDEO_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/dance_1723_video_psi0/dance_1723_video_10s
|
| 28 |
+
PSI_DANCE_VIDEO_EMBED_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/dance_1723_video_psi0/dance_1723_video_10s_embeds
|
| 29 |
+
PSI_VIDEO_DEMO_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/videogen/motion_example_0506_demo
|
| 30 |
+
PSI_QWEN3_TTS_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/qwen3-tts
|
| 31 |
+
PSI_VIDEO_CLIP_DATASET_ROOT=${PSI_VIDEO_CLIP_DATASET_ROOT:-/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/videogen/psi0_video_clip_50f_dense_stride1_direct_full}
|
| 32 |
+
PSI_DANCE_VIDEO_CLIP_ROOT=$PSI_VIDEO_CLIP_DATASET_ROOT/dance_1723_video_10s
|
| 33 |
+
PSI_MOTION_EXAMPLE_VIDEO_CLIP_ROOT=$PSI_VIDEO_CLIP_DATASET_ROOT/motion_example_0506_demo
|
| 34 |
+
PSI_DANCE_VIDEO_CLIP_EMBED_ROOT=${PSI_DANCE_VIDEO_CLIP_EMBED_ROOT:-$PSI_DANCE_VIDEO_CLIP_ROOT/embeds}
|
| 35 |
+
PSI_MOTION_EXAMPLE_VIDEO_CLIP_EMBED_ROOT=${PSI_MOTION_EXAMPLE_VIDEO_CLIP_EMBED_ROOT:-$PSI_MOTION_EXAMPLE_VIDEO_CLIP_ROOT/embeds}
|
| 36 |
+
PSI_MOTION_EXAMPLE_VIDEO_EMBED_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/videogen/motion_example_0506_demo_video_embeds
|
| 37 |
+
|
| 38 |
+
if [ -z "$PSI_AUDIO_SOURCES_JSON" ]; then
|
| 39 |
+
PSI_AUDIO_SOURCES_JSON=$(compact_json <<JSON
|
| 40 |
+
[
|
| 41 |
+
{
|
| 42 |
+
"name": "psi0_dance_audio",
|
| 43 |
+
"root_dir": "$PSI_DANCE_MOTION_ROOT",
|
| 44 |
+
"video_dir": "$PSI_DANCE_AUDIO_VIDEO_ROOT",
|
| 45 |
+
"dataset_name": "seed-audio-g1"
|
| 46 |
+
}
|
| 47 |
+
]
|
| 48 |
+
JSON
|
| 49 |
+
)
|
| 50 |
+
fi
|
| 51 |
+
|
| 52 |
+
if [ -z "$PSI_AUDIO_COMMAND_SOURCES_JSON" ]; then
|
| 53 |
+
PSI_AUDIO_COMMAND_SOURCES_JSON=$(compact_json <<JSON
|
| 54 |
+
[
|
| 55 |
+
{
|
| 56 |
+
"name": "qwen3_tts_seed_audio_command",
|
| 57 |
+
"root_dir": "$PSI_SEED_ROOT",
|
| 58 |
+
"dataset_root": "$PSI_QWEN3_TTS_ROOT",
|
| 59 |
+
"metadata_jsonl": "metadata.jsonl",
|
| 60 |
+
"motion_dir_key": "filename",
|
| 61 |
+
"audio_path_key": "audio_path",
|
| 62 |
+
"instruction_key": "content_short_description",
|
| 63 |
+
"motion_metadata_csv": "$PSI_SEED_METADATA",
|
| 64 |
+
"motion_metadata_key": "filename",
|
| 65 |
+
"motion_metadata_path_key": "move_g1_path",
|
| 66 |
+
"audio_file_extensions": ".wav",
|
| 67 |
+
"audio_task": "command",
|
| 68 |
+
"dataset_name": "seed-audio-command-g1"
|
| 69 |
+
}
|
| 70 |
+
]
|
| 71 |
+
JSON
|
| 72 |
+
)
|
| 73 |
+
fi
|
| 74 |
+
|
| 75 |
+
if [ -z "$PSI_VIDEO_SOURCES_JSON" ]; then
|
| 76 |
+
PSI_VIDEO_SOURCES_JSON=$(compact_json <<JSON
|
| 77 |
+
[
|
| 78 |
+
{
|
| 79 |
+
"name": "psi0_dance_gvhmr",
|
| 80 |
+
"root_dir": "$PSI_DANCE_MOTION_ROOT",
|
| 81 |
+
"video_dir": "$PSI_DANCE_VIDEO_ROOT",
|
| 82 |
+
"video_embed_dir": "$PSI_DANCE_VIDEO_EMBED_ROOT",
|
| 83 |
+
"dataset_name": "seed-video-g1"
|
| 84 |
+
},
|
| 85 |
+
{
|
| 86 |
+
"name": "motion_example_0506_demo",
|
| 87 |
+
"dataset_root": "$PSI_VIDEO_DEMO_ROOT",
|
| 88 |
+
"metadata_jsonl": "metadata.jsonl",
|
| 89 |
+
"motion_dir_key": "converted_path",
|
| 90 |
+
"video_path_key": "video_path",
|
| 91 |
+
"instruction_key": "video_caption",
|
| 92 |
+
"video_embed_dir": "$PSI_MOTION_EXAMPLE_VIDEO_EMBED_ROOT",
|
| 93 |
+
"dataset_name": "motion-example-video-g1"
|
| 94 |
+
}
|
| 95 |
+
]
|
| 96 |
+
JSON
|
| 97 |
+
)
|
| 98 |
+
fi
|
| 99 |
+
|
| 100 |
+
if [ -z "$PSI_VIDEO_CLIP_SOURCES_JSON" ]; then
|
| 101 |
+
PSI_VIDEO_CLIP_SOURCES_JSON=$(compact_json <<JSON
|
| 102 |
+
[
|
| 103 |
+
{
|
| 104 |
+
"name": "dance_1723_video_10s_clip50",
|
| 105 |
+
"dataset_root": "$PSI_DANCE_VIDEO_CLIP_ROOT",
|
| 106 |
+
"metadata_jsonl": "metadata.jsonl",
|
| 107 |
+
"motion_dir_key": "converted_path",
|
| 108 |
+
"video_path_key": "video_path",
|
| 109 |
+
"instruction_key": "video_caption",
|
| 110 |
+
"video_embed_dir": "$PSI_DANCE_VIDEO_CLIP_EMBED_ROOT",
|
| 111 |
+
"dataset_name": "dance-video-clip-g1"
|
| 112 |
+
},
|
| 113 |
+
{
|
| 114 |
+
"name": "motion_example_0506_demo_clip50",
|
| 115 |
+
"dataset_root": "$PSI_MOTION_EXAMPLE_VIDEO_CLIP_ROOT",
|
| 116 |
+
"metadata_jsonl": "metadata.jsonl",
|
| 117 |
+
"motion_dir_key": "converted_path",
|
| 118 |
+
"video_path_key": "video_path",
|
| 119 |
+
"instruction_key": "video_caption",
|
| 120 |
+
"video_embed_dir": "$PSI_MOTION_EXAMPLE_VIDEO_CLIP_EMBED_ROOT",
|
| 121 |
+
"dataset_name": "motion-example-video-clip-g1"
|
| 122 |
+
}
|
| 123 |
+
]
|
| 124 |
+
JSON
|
| 125 |
+
)
|
| 126 |
+
fi
|
| 127 |
+
|
| 128 |
+
PSI_TEXT_TASK_WEIGHT=0
|
| 129 |
+
PSI_AUDIO_TEXT_TASK_WEIGHT=0
|
| 130 |
+
PSI_AUDIO_ONLY_TASK_WEIGHT=0
|
| 131 |
+
PSI_AUDIO_COMMAND_TEXT_TASK_WEIGHT=0
|
| 132 |
+
PSI_AUDIO_COMMAND_ONLY_TASK_WEIGHT=0
|
| 133 |
+
PSI_VIDEO_TEXT_TASK_WEIGHT=0
|
| 134 |
+
PSI_VIDEO_ONLY_TASK_WEIGHT=0
|
| 135 |
+
PSI_VIDEO_CLIP_TEXT_TASK_WEIGHT=0.5
|
| 136 |
+
PSI_VIDEO_CLIP_ONLY_TASK_WEIGHT=0.5
|
| 137 |
+
|
| 138 |
+
PSI_PRETRAIN_BACKBONE_PATH=/cpfs01/VideoGen/public_ckpt/Qwen/Qwen2.5-Omni-3B
|
| 139 |
+
NUM_PAST_FRAMES=0
|
| 140 |
+
STATE_HORIZON=$((NUM_PAST_FRAMES + 1))
|
| 141 |
+
ACTION_CHUNK_SIZE=30
|
| 142 |
+
ACTION_EXEC_HORIZON=20
|
| 143 |
+
PROGRESS_DIM=5
|
| 144 |
+
VIDEO_SAMPLE_FPS=1.0
|
| 145 |
+
VIDEO_CLIP_SAMPLE_FPS=8.0
|
| 146 |
+
VIDEO_SIZE_PIXELS=50176
|
| 147 |
+
VIDEO_DECODER=ffmpeg
|
| 148 |
+
VIDEO_CACHE_SIZE=64
|
| 149 |
+
VIDEO_CLIP_MOTION_CACHE_SIZE=256
|
| 150 |
+
VIDEO_CLIP_TRUST_METADATA=${VIDEO_CLIP_TRUST_METADATA:-1}
|
| 151 |
+
VIDEO_CLIP_RECORDS_CACHE_DIR=${VIDEO_CLIP_RECORDS_CACHE_DIR:-$PSI_VIDEO_CLIP_DATASET_ROOT/.index_cache}
|
| 152 |
+
if [ "$VIDEO_CLIP_TRUST_METADATA" = "1" ] || [ "$VIDEO_CLIP_TRUST_METADATA" = "true" ] || [ "$VIDEO_CLIP_TRUST_METADATA" = "True" ]; then
|
| 153 |
+
VIDEO_CLIP_TRUST_METADATA_FLAG="--data.video-clip.trust-metadata"
|
| 154 |
+
else
|
| 155 |
+
VIDEO_CLIP_TRUST_METADATA_FLAG="--data.video-clip.no-trust-metadata"
|
| 156 |
+
fi
|
| 157 |
+
DATALOADER_NUM_WORKERS=4
|
| 158 |
+
VAL_DATALOADER_NUM_WORKERS=2
|
| 159 |
+
|
| 160 |
+
NPROC_PER_NODE=$(echo $CUDA_VISIBLE_DEVICES | tr ',' '\n' | wc -l)
|
| 161 |
+
|
| 162 |
+
echo "Training with $NPROC_PER_NODE GPU(s): $CUDA_VISIBLE_DEVICES"
|
| 163 |
+
echo "Text root: $PSI_SEED_ROOT"
|
| 164 |
+
echo "Dance motion root: $PSI_DANCE_MOTION_ROOT"
|
| 165 |
+
echo "Dance audio/video root: $PSI_DANCE_AUDIO_VIDEO_ROOT"
|
| 166 |
+
echo "Dance video root: $PSI_DANCE_VIDEO_ROOT"
|
| 167 |
+
echo "Dance video embed root: $PSI_DANCE_VIDEO_EMBED_ROOT"
|
| 168 |
+
echo "Qwen3-TTS acdudio command root: $PSI_QWEN3_TTS_ROOT"
|
| 169 |
+
echo "Audio sources: $PSI_AUDIO_SOURCES_JSON"
|
| 170 |
+
echo "Audio command sources: $PSI_AUDIO_COMMAND_SOURCES_JSON"
|
| 171 |
+
echo "Video sources: $PSI_VIDEO_SOURCES_JSON"
|
| 172 |
+
echo "Video clip sources: $PSI_VIDEO_CLIP_SOURCES_JSON"
|
| 173 |
+
echo "Task weights: text=$PSI_TEXT_TASK_WEIGHT audio_text=$PSI_AUDIO_TEXT_TASK_WEIGHT audio_only=$PSI_AUDIO_ONLY_TASK_WEIGHT audio_command_text=$PSI_AUDIO_COMMAND_TEXT_TASK_WEIGHT audio_command_only=$PSI_AUDIO_COMMAND_ONLY_TASK_WEIGHT video_text=$PSI_VIDEO_TEXT_TASK_WEIGHT video_only=$PSI_VIDEO_ONLY_TASK_WEIGHT video_clip_text=$PSI_VIDEO_CLIP_TEXT_TASK_WEIGHT video_clip_only=$PSI_VIDEO_CLIP_ONLY_TASK_WEIGHT"
|
| 174 |
+
# echo "Resume run dir: $PSI_PRETRAIN_RUN_DIR"
|
| 175 |
+
echo "TextOp 39D action/state enabled"
|
| 176 |
+
echo "Future progress embedding: enabled, dim $PROGRESS_DIM"
|
| 177 |
+
echo "Backbone path: $PSI_PRETRAIN_BACKBONE_PATH"
|
| 178 |
+
echo "Output dir: $PSI_OUTPUT_DIR"
|
| 179 |
+
echo "Video clip trust metadata: $VIDEO_CLIP_TRUST_METADATA"
|
| 180 |
+
echo "Video clip records cache dir: $VIDEO_CLIP_RECORDS_CACHE_DIR"
|
| 181 |
+
|
| 182 |
+
args="posttrain_v1_seed_text_audio_video_mix_psi0_config \
|
| 183 |
+
--seed=292285 \
|
| 184 |
+
--exp=posttrain \
|
| 185 |
+
--timestamp=$(date +"%y%m%d%H%M") \
|
| 186 |
+
--train.name=posttrain_v1_trimodal \
|
| 187 |
+
--train.data_parallel=ddp \
|
| 188 |
+
--train.mixed_precision=bf16 \
|
| 189 |
+
--train.train_batch_size=32 \
|
| 190 |
+
--train.val_batch_size=64 \
|
| 191 |
+
--train.num-workers=$DATALOADER_NUM_WORKERS \
|
| 192 |
+
--train.val-num-workers=$VAL_DATALOADER_NUM_WORKERS \
|
| 193 |
+
--train.resume_from_checkpoint=None \
|
| 194 |
+
--train.max_checkpoints_to_keep=1 \
|
| 195 |
+
--train.gradient_accumulation_steps=1 \
|
| 196 |
+
--train.learning_rate=1e-4 \
|
| 197 |
+
--train.max_training_steps=500000 \
|
| 198 |
+
--train.warmup_ratio=None \
|
| 199 |
+
--train.warmup_steps=1000 \
|
| 200 |
+
--train.checkpointing_steps=1000 \
|
| 201 |
+
--train.validation_steps=1000 \
|
| 202 |
+
--train.val_num_batches=20 \
|
| 203 |
+
--train.max_grad_norm=1.0 \
|
| 204 |
+
--train.lr_scheduler_type=cosine_with_min_lr \
|
| 205 |
+
--train.lr_scheduler_kwargs.betas 0.9 0.999 \
|
| 206 |
+
--train.lr_scheduler_kwargs.weight_decay=0 \
|
| 207 |
+
--train.lr_scheduler_kwargs.eps=1e-8 \
|
| 208 |
+
--log.report_to=$PSI_LOG_REPORT_TO \
|
| 209 |
+
--train.output_dir=$PSI_OUTPUT_DIR \
|
| 210 |
+
--data.text.root-dir=$PSI_SEED_ROOT \
|
| 211 |
+
--data.text.metadata-csv=$PSI_SEED_METADATA \
|
| 212 |
+
--data.text-ratio=$PSI_TEXT_TASK_WEIGHT \
|
| 213 |
+
--data.audio.sources-json=$PSI_AUDIO_SOURCES_JSON \
|
| 214 |
+
--data.audio-ratio=$PSI_AUDIO_TEXT_TASK_WEIGHT \
|
| 215 |
+
--data.audio-only-ratio=$PSI_AUDIO_ONLY_TASK_WEIGHT \
|
| 216 |
+
--data.audio.sample-stride=1 \
|
| 217 |
+
--data.audio.motion-fps=50 \
|
| 218 |
+
--data.audio.audio-sampling-rate=16000 \
|
| 219 |
+
--data.audio.audio-window-seconds=1.0 \
|
| 220 |
+
--data.audio.audio-left-context-seconds=0.0 \
|
| 221 |
+
--data.audio.full-audio \
|
| 222 |
+
--data.audio-command.sources-json=$PSI_AUDIO_COMMAND_SOURCES_JSON \
|
| 223 |
+
--data.audio-command-ratio=$PSI_AUDIO_COMMAND_TEXT_TASK_WEIGHT \
|
| 224 |
+
--data.audio-command-only-ratio=$PSI_AUDIO_COMMAND_ONLY_TASK_WEIGHT \
|
| 225 |
+
--data.audio-command.sample-stride=1 \
|
| 226 |
+
--data.audio-command.motion-fps=50 \
|
| 227 |
+
--data.audio-command.audio-sampling-rate=16000 \
|
| 228 |
+
--data.audio-command.audio-window-seconds=1.0 \
|
| 229 |
+
--data.audio-command.audio-left-context-seconds=0.0 \
|
| 230 |
+
--data.audio-command.full-audio \
|
| 231 |
+
--data.video-ratio=$PSI_VIDEO_TEXT_TASK_WEIGHT \
|
| 232 |
+
--data.video-only-ratio=$PSI_VIDEO_ONLY_TASK_WEIGHT \
|
| 233 |
+
--data.video.sample-stride=1 \
|
| 234 |
+
--data.video.motion-fps=50 \
|
| 235 |
+
--data.video.video-sample-fps=$VIDEO_SAMPLE_FPS \
|
| 236 |
+
--data.video.video-size-pixels=$VIDEO_SIZE_PIXELS \
|
| 237 |
+
--data.video.video-decoder=$VIDEO_DECODER \
|
| 238 |
+
--data.video.video-cache-size=$VIDEO_CACHE_SIZE \
|
| 239 |
+
--data.video.sources-json=$PSI_VIDEO_SOURCES_JSON \
|
| 240 |
+
--data.video-clip-text-ratio=$PSI_VIDEO_CLIP_TEXT_TASK_WEIGHT \
|
| 241 |
+
--data.video-clip-only-ratio=$PSI_VIDEO_CLIP_ONLY_TASK_WEIGHT \
|
| 242 |
+
--data.video-clip.sources-json=$PSI_VIDEO_CLIP_SOURCES_JSON \
|
| 243 |
+
--data.video-clip.sample-stride=1 \
|
| 244 |
+
--data.video-clip.motion-fps=50 \
|
| 245 |
+
--data.video-clip.video-sample-fps=$VIDEO_CLIP_SAMPLE_FPS \
|
| 246 |
+
--data.video-clip.video-size-pixels=$VIDEO_SIZE_PIXELS \
|
| 247 |
+
--data.video-clip.video-decoder=$VIDEO_DECODER \
|
| 248 |
+
--data.video-clip.video-cache-size=$VIDEO_CACHE_SIZE \
|
| 249 |
+
--data.video-clip.motion-cache-size=$VIDEO_CLIP_MOTION_CACHE_SIZE \
|
| 250 |
+
$VIDEO_CLIP_TRUST_METADATA_FLAG \
|
| 251 |
+
--data.video-clip.records-cache-dir=$VIDEO_CLIP_RECORDS_CACHE_DIR \
|
| 252 |
+
--data.video-clip.action-exec-horizon=$ACTION_EXEC_HORIZON \
|
| 253 |
+
--data.sampler=batch_mixture \
|
| 254 |
+
--data.val-task=video_clip,video_clip_only \
|
| 255 |
+
--data.transform.repack.num-past-frames=$NUM_PAST_FRAMES \
|
| 256 |
+
--data.transform.repack.textop-actions \
|
| 257 |
+
--data.transform.repack.action-chunk-size=$ACTION_CHUNK_SIZE \
|
| 258 |
+
--data.transform.repack.pad-action-dim=39 \
|
| 259 |
+
--data.transform.repack.pad-state-dim=39 \
|
| 260 |
+
--data.transform.field.action_norm_type=bounds_q99 \
|
| 261 |
+
--data.transform.field.stat-path=assets/stats/seed_all_g1_motion39_textop_stats.json \
|
| 262 |
+
--data.transform.field.normalize-state \
|
| 263 |
+
--data.transform.field.pad-action-dim=39 \
|
| 264 |
+
--data.transform.field.pad-state-dim=39 \
|
| 265 |
+
--data.transform.model.resize.size 240 320 \
|
| 266 |
+
--data.transform.model.center_crop.size 240 320 \
|
| 267 |
+
--data.transform.model.no-img-aug \
|
| 268 |
+
--data.transform.model.text-only \
|
| 269 |
+
--data.transform.model.audio-only \
|
| 270 |
+
--data.transform.model.video-only \
|
| 271 |
+
--data.transform.model.video-size-pixels=$VIDEO_SIZE_PIXELS \
|
| 272 |
+
--model.model_name_or_path=$PSI_PRETRAIN_BACKBONE_PATH \
|
| 273 |
+
--model.noise-scheduler=flow \
|
| 274 |
+
--model.n_conditions=0 \
|
| 275 |
+
--model.action-chunk-size=$ACTION_CHUNK_SIZE \
|
| 276 |
+
--model.action-dim=39 \
|
| 277 |
+
--model.action-exec-horizon=$ACTION_EXEC_HORIZON \
|
| 278 |
+
--model.observation-horizon=$STATE_HORIZON \
|
| 279 |
+
--model.odim=39 \
|
| 280 |
+
--model.view_feature_dim=2048 \
|
| 281 |
+
--model.use-future-progress-embedding \
|
| 282 |
+
--model.future-progress-dim=$PROGRESS_DIM \
|
| 283 |
+
--model.tune-vlm \
|
| 284 |
+
--model.lang-backbone-lr=1e-5 \
|
| 285 |
+
--model.no-use_film \
|
| 286 |
+
--model.no-combined_temb \
|
| 287 |
+
--model.rtc \
|
| 288 |
+
--model.max-delay=11 \
|
| 289 |
+
--train.max-grad-value=1 \
|
| 290 |
+
--train.loss-group-weight.joint-abs=1.0 \
|
| 291 |
+
--train.loss-group-weight.root-delta-xy=3.0 \
|
| 292 |
+
--train.loss-group-weight.root-rot5-delta=10.0 \
|
| 293 |
+
--train.loss-group-weight.root-height=15.0 \
|
| 294 |
+
--train.loss-group-weight.foot-contact=0.1
|
| 295 |
+
"
|
| 296 |
+
torchrun --nproc_per_node=$NPROC_PER_NODE --master_port=29500 scripts/train.py \
|
| 297 |
+
$args "$@"
|
posttrain-v1-psi0-all-tune-vlm.sh
ADDED
|
@@ -0,0 +1,282 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
pkill -f monitor.py
|
| 2 |
+
|
| 3 |
+
cd /cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/Psi0-yth-v18
|
| 4 |
+
export PYTHONPATH=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/Psi0-yth-v18/src
|
| 5 |
+
export CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7
|
| 6 |
+
export PSI_OUTPUT_DIR=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/Psi0-yth-v18/runs
|
| 7 |
+
export PSI_LOG_REPORT_TO=wandb
|
| 8 |
+
|
| 9 |
+
source .venv-psi/bin/activate
|
| 10 |
+
[ -f .env ] && source .env
|
| 11 |
+
|
| 12 |
+
export WANDB_API_KEY="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"
|
| 13 |
+
|
| 14 |
+
export OMP_NUM_THREADS=8
|
| 15 |
+
export TF_CPP_MIN_LOG_LEVEL=3
|
| 16 |
+
|
| 17 |
+
# Keep JSON sources readable in this script, but pass compact JSON to tyro.
|
| 18 |
+
compact_json() {
|
| 19 |
+
python -c 'import json, sys; print(json.dumps(json.load(sys.stdin), separators=(",", ":")))'
|
| 20 |
+
}
|
| 21 |
+
|
| 22 |
+
PSI_SEED_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/Psi0/dataset/seed_all
|
| 23 |
+
PSI_SEED_METADATA=metadata/seed_metadata_v004.csv
|
| 24 |
+
|
| 25 |
+
PSI_DANCE_MOTION_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/GR00T-WholeBodyControl/gear_sonic_deploy/reference/converted/psi0_dance/gvhmr
|
| 26 |
+
PSI_DANCE_AUDIO_VIDEO_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/dance_1723_video_psi0/dance_1723_video_10s
|
| 27 |
+
PSI_DANCE_VIDEO_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/dance_1723_video_psi0/dance_1723_video_10s
|
| 28 |
+
PSI_DANCE_VIDEO_EMBED_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/dance_1723_video_psi0/dance_1723_video_10s_embeds
|
| 29 |
+
PSI_VIDEO_DEMO_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/videogen/motion_example_0506_demo
|
| 30 |
+
PSI_QWEN3_TTS_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/qwen3-tts
|
| 31 |
+
PSI_VIDEO_CLIP_DATASET_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/videogen/psi0_video_clip_50f_rand8_full
|
| 32 |
+
PSI_DANCE_VIDEO_CLIP_ROOT=$PSI_VIDEO_CLIP_DATASET_ROOT/dance_1723_video_10s
|
| 33 |
+
PSI_MOTION_EXAMPLE_VIDEO_CLIP_ROOT=$PSI_VIDEO_CLIP_DATASET_ROOT/motion_example_0506_demo
|
| 34 |
+
PSI_MOTION_EXAMPLE_VIDEO_EMBED_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/videogen/motion_example_0506_demo_video_embeds
|
| 35 |
+
|
| 36 |
+
if [ -z "$PSI_AUDIO_SOURCES_JSON" ]; then
|
| 37 |
+
PSI_AUDIO_SOURCES_JSON=$(compact_json <<JSON
|
| 38 |
+
[
|
| 39 |
+
{
|
| 40 |
+
"name": "psi0_dance_audio",
|
| 41 |
+
"root_dir": "$PSI_DANCE_MOTION_ROOT",
|
| 42 |
+
"video_dir": "$PSI_DANCE_AUDIO_VIDEO_ROOT",
|
| 43 |
+
"dataset_name": "seed-audio-g1"
|
| 44 |
+
}
|
| 45 |
+
]
|
| 46 |
+
JSON
|
| 47 |
+
)
|
| 48 |
+
fi
|
| 49 |
+
|
| 50 |
+
if [ -z "$PSI_AUDIO_COMMAND_SOURCES_JSON" ]; then
|
| 51 |
+
PSI_AUDIO_COMMAND_SOURCES_JSON=$(compact_json <<JSON
|
| 52 |
+
[
|
| 53 |
+
{
|
| 54 |
+
"name": "qwen3_tts_seed_audio_command",
|
| 55 |
+
"root_dir": "$PSI_SEED_ROOT",
|
| 56 |
+
"dataset_root": "$PSI_QWEN3_TTS_ROOT",
|
| 57 |
+
"metadata_jsonl": "metadata.jsonl",
|
| 58 |
+
"motion_dir_key": "filename",
|
| 59 |
+
"audio_path_key": "audio_path",
|
| 60 |
+
"instruction_key": "content_short_description",
|
| 61 |
+
"motion_metadata_csv": "$PSI_SEED_METADATA",
|
| 62 |
+
"motion_metadata_key": "filename",
|
| 63 |
+
"motion_metadata_path_key": "move_g1_path",
|
| 64 |
+
"audio_file_extensions": ".wav",
|
| 65 |
+
"audio_task": "command",
|
| 66 |
+
"dataset_name": "seed-audio-command-g1"
|
| 67 |
+
}
|
| 68 |
+
]
|
| 69 |
+
JSON
|
| 70 |
+
)
|
| 71 |
+
fi
|
| 72 |
+
|
| 73 |
+
if [ -z "$PSI_VIDEO_SOURCES_JSON" ]; then
|
| 74 |
+
PSI_VIDEO_SOURCES_JSON=$(compact_json <<JSON
|
| 75 |
+
[
|
| 76 |
+
{
|
| 77 |
+
"name": "psi0_dance_gvhmr",
|
| 78 |
+
"root_dir": "$PSI_DANCE_MOTION_ROOT",
|
| 79 |
+
"video_dir": "$PSI_DANCE_VIDEO_ROOT",
|
| 80 |
+
"video_embed_dir": "$PSI_DANCE_VIDEO_EMBED_ROOT",
|
| 81 |
+
"dataset_name": "seed-video-g1"
|
| 82 |
+
},
|
| 83 |
+
{
|
| 84 |
+
"name": "motion_example_0506_demo",
|
| 85 |
+
"dataset_root": "$PSI_VIDEO_DEMO_ROOT",
|
| 86 |
+
"metadata_jsonl": "metadata.jsonl",
|
| 87 |
+
"motion_dir_key": "converted_path",
|
| 88 |
+
"video_path_key": "video_path",
|
| 89 |
+
"instruction_key": "video_caption",
|
| 90 |
+
"video_embed_dir": "$PSI_MOTION_EXAMPLE_VIDEO_EMBED_ROOT",
|
| 91 |
+
"dataset_name": "motion-example-video-g1"
|
| 92 |
+
}
|
| 93 |
+
]
|
| 94 |
+
JSON
|
| 95 |
+
)
|
| 96 |
+
fi
|
| 97 |
+
|
| 98 |
+
if [ -z "$PSI_VIDEO_CLIP_SOURCES_JSON" ]; then
|
| 99 |
+
PSI_VIDEO_CLIP_SOURCES_JSON=$(compact_json <<JSON
|
| 100 |
+
[
|
| 101 |
+
{
|
| 102 |
+
"name": "dance_1723_video_10s_clip50",
|
| 103 |
+
"dataset_root": "$PSI_DANCE_VIDEO_CLIP_ROOT",
|
| 104 |
+
"metadata_jsonl": "metadata.jsonl",
|
| 105 |
+
"motion_dir_key": "converted_path",
|
| 106 |
+
"video_path_key": "video_path",
|
| 107 |
+
"instruction_key": "video_caption",
|
| 108 |
+
"video_embed_dir": "$PSI_DANCE_VIDEO_CLIP_ROOT/embeds",
|
| 109 |
+
"dataset_name": "dance-video-clip-g1"
|
| 110 |
+
},
|
| 111 |
+
{
|
| 112 |
+
"name": "motion_example_0506_demo_clip50",
|
| 113 |
+
"dataset_root": "$PSI_MOTION_EXAMPLE_VIDEO_CLIP_ROOT",
|
| 114 |
+
"metadata_jsonl": "metadata.jsonl",
|
| 115 |
+
"motion_dir_key": "converted_path",
|
| 116 |
+
"video_path_key": "video_path",
|
| 117 |
+
"instruction_key": "video_caption",
|
| 118 |
+
"video_embed_dir": "$PSI_MOTION_EXAMPLE_VIDEO_CLIP_ROOT/embeds",
|
| 119 |
+
"dataset_name": "motion-example-video-clip-g1"
|
| 120 |
+
}
|
| 121 |
+
]
|
| 122 |
+
JSON
|
| 123 |
+
)
|
| 124 |
+
fi
|
| 125 |
+
|
| 126 |
+
PSI_TEXT_TASK_WEIGHT=0
|
| 127 |
+
PSI_AUDIO_TEXT_TASK_WEIGHT=0
|
| 128 |
+
PSI_AUDIO_ONLY_TASK_WEIGHT=0
|
| 129 |
+
PSI_AUDIO_COMMAND_TEXT_TASK_WEIGHT=0
|
| 130 |
+
PSI_AUDIO_COMMAND_ONLY_TASK_WEIGHT=0
|
| 131 |
+
PSI_VIDEO_TEXT_TASK_WEIGHT=0
|
| 132 |
+
PSI_VIDEO_ONLY_TASK_WEIGHT=0
|
| 133 |
+
PSI_VIDEO_CLIP_TEXT_TASK_WEIGHT=0.5
|
| 134 |
+
PSI_VIDEO_CLIP_ONLY_TASK_WEIGHT=0.5
|
| 135 |
+
|
| 136 |
+
PSI_PRETRAIN_BACKBONE_PATH=/cpfs01/VideoGen/public_ckpt/Qwen/Qwen2.5-Omni-3B
|
| 137 |
+
NUM_PAST_FRAMES=0
|
| 138 |
+
STATE_HORIZON=$((NUM_PAST_FRAMES + 1))
|
| 139 |
+
ACTION_CHUNK_SIZE=30
|
| 140 |
+
ACTION_EXEC_HORIZON=20
|
| 141 |
+
PROGRESS_DIM=5
|
| 142 |
+
VIDEO_SAMPLE_FPS=1.0
|
| 143 |
+
VIDEO_CLIP_SAMPLE_FPS=8.0
|
| 144 |
+
VIDEO_SIZE_PIXELS=50176
|
| 145 |
+
VIDEO_DECODER=ffmpeg
|
| 146 |
+
VIDEO_CACHE_SIZE=64
|
| 147 |
+
DATALOADER_NUM_WORKERS=4
|
| 148 |
+
VAL_DATALOADER_NUM_WORKERS=2
|
| 149 |
+
|
| 150 |
+
NPROC_PER_NODE=$(echo $CUDA_VISIBLE_DEVICES | tr ',' '\n' | wc -l)
|
| 151 |
+
|
| 152 |
+
echo "Training with $NPROC_PER_NODE GPU(s): $CUDA_VISIBLE_DEVICES"
|
| 153 |
+
echo "Text root: $PSI_SEED_ROOT"
|
| 154 |
+
echo "Dance motion root: $PSI_DANCE_MOTION_ROOT"
|
| 155 |
+
echo "Dance audio/video root: $PSI_DANCE_AUDIO_VIDEO_ROOT"
|
| 156 |
+
echo "Dance video root: $PSI_DANCE_VIDEO_ROOT"
|
| 157 |
+
echo "Dance video embed root: $PSI_DANCE_VIDEO_EMBED_ROOT"
|
| 158 |
+
echo "Qwen3-TTS acdudio command root: $PSI_QWEN3_TTS_ROOT"
|
| 159 |
+
echo "Audio sources: $PSI_AUDIO_SOURCES_JSON"
|
| 160 |
+
echo "Audio command sources: $PSI_AUDIO_COMMAND_SOURCES_JSON"
|
| 161 |
+
echo "Video sources: $PSI_VIDEO_SOURCES_JSON"
|
| 162 |
+
echo "Video clip sources: $PSI_VIDEO_CLIP_SOURCES_JSON"
|
| 163 |
+
echo "Task weights: text=$PSI_TEXT_TASK_WEIGHT audio_text=$PSI_AUDIO_TEXT_TASK_WEIGHT audio_only=$PSI_AUDIO_ONLY_TASK_WEIGHT audio_command_text=$PSI_AUDIO_COMMAND_TEXT_TASK_WEIGHT audio_command_only=$PSI_AUDIO_COMMAND_ONLY_TASK_WEIGHT video_text=$PSI_VIDEO_TEXT_TASK_WEIGHT video_only=$PSI_VIDEO_ONLY_TASK_WEIGHT video_clip_text=$PSI_VIDEO_CLIP_TEXT_TASK_WEIGHT video_clip_only=$PSI_VIDEO_CLIP_ONLY_TASK_WEIGHT"
|
| 164 |
+
# echo "Resume run dir: $PSI_PRETRAIN_RUN_DIR"
|
| 165 |
+
echo "TextOp 39D action/state enabled"
|
| 166 |
+
echo "Future progress embedding: enabled, dim $PROGRESS_DIM"
|
| 167 |
+
echo "Backbone path: $PSI_PRETRAIN_BACKBONE_PATH"
|
| 168 |
+
echo "Output dir: $PSI_OUTPUT_DIR"
|
| 169 |
+
|
| 170 |
+
args="posttrain_v1_seed_text_audio_video_mix_psi0_config \
|
| 171 |
+
--seed=292285 \
|
| 172 |
+
--exp=posttrain \
|
| 173 |
+
--timestamp=$(date +"%y%m%d%H%M") \
|
| 174 |
+
--train.name=posttrain_v1_trimodal \
|
| 175 |
+
--train.data_parallel=ddp \
|
| 176 |
+
--train.mixed_precision=bf16 \
|
| 177 |
+
--train.train_batch_size=32 \
|
| 178 |
+
--train.val_batch_size=64 \
|
| 179 |
+
--train.num-workers=$DATALOADER_NUM_WORKERS \
|
| 180 |
+
--train.val-num-workers=$VAL_DATALOADER_NUM_WORKERS \
|
| 181 |
+
--train.resume_from_checkpoint=None \
|
| 182 |
+
--train.max_checkpoints_to_keep=1 \
|
| 183 |
+
--train.gradient_accumulation_steps=1 \
|
| 184 |
+
--train.learning_rate=1e-4 \
|
| 185 |
+
--train.max_training_steps=500000 \
|
| 186 |
+
--train.warmup_ratio=None \
|
| 187 |
+
--train.warmup_steps=1000 \
|
| 188 |
+
--train.checkpointing_steps=1000 \
|
| 189 |
+
--train.validation_steps=1000 \
|
| 190 |
+
--train.val_num_batches=20 \
|
| 191 |
+
--train.max_grad_norm=1.0 \
|
| 192 |
+
--train.lr_scheduler_type=cosine_with_min_lr \
|
| 193 |
+
--train.lr_scheduler_kwargs.betas 0.9 0.999 \
|
| 194 |
+
--train.lr_scheduler_kwargs.weight_decay=0 \
|
| 195 |
+
--train.lr_scheduler_kwargs.eps=1e-8 \
|
| 196 |
+
--log.report_to=$PSI_LOG_REPORT_TO \
|
| 197 |
+
--train.output_dir=$PSI_OUTPUT_DIR \
|
| 198 |
+
--data.text.root-dir=$PSI_SEED_ROOT \
|
| 199 |
+
--data.text.metadata-csv=$PSI_SEED_METADATA \
|
| 200 |
+
--data.text-ratio=$PSI_TEXT_TASK_WEIGHT \
|
| 201 |
+
--data.audio.sources-json=$PSI_AUDIO_SOURCES_JSON \
|
| 202 |
+
--data.audio-ratio=$PSI_AUDIO_TEXT_TASK_WEIGHT \
|
| 203 |
+
--data.audio-only-ratio=$PSI_AUDIO_ONLY_TASK_WEIGHT \
|
| 204 |
+
--data.audio.sample-stride=1 \
|
| 205 |
+
--data.audio.motion-fps=50 \
|
| 206 |
+
--data.audio.audio-sampling-rate=16000 \
|
| 207 |
+
--data.audio.audio-window-seconds=1.0 \
|
| 208 |
+
--data.audio.audio-left-context-seconds=0.0 \
|
| 209 |
+
--data.audio.full-audio \
|
| 210 |
+
--data.audio-command.sources-json=$PSI_AUDIO_COMMAND_SOURCES_JSON \
|
| 211 |
+
--data.audio-command-ratio=$PSI_AUDIO_COMMAND_TEXT_TASK_WEIGHT \
|
| 212 |
+
--data.audio-command-only-ratio=$PSI_AUDIO_COMMAND_ONLY_TASK_WEIGHT \
|
| 213 |
+
--data.audio-command.sample-stride=1 \
|
| 214 |
+
--data.audio-command.motion-fps=50 \
|
| 215 |
+
--data.audio-command.audio-sampling-rate=16000 \
|
| 216 |
+
--data.audio-command.audio-window-seconds=1.0 \
|
| 217 |
+
--data.audio-command.audio-left-context-seconds=0.0 \
|
| 218 |
+
--data.audio-command.full-audio \
|
| 219 |
+
--data.video-ratio=$PSI_VIDEO_TEXT_TASK_WEIGHT \
|
| 220 |
+
--data.video-only-ratio=$PSI_VIDEO_ONLY_TASK_WEIGHT \
|
| 221 |
+
--data.video.sample-stride=1 \
|
| 222 |
+
--data.video.motion-fps=50 \
|
| 223 |
+
--data.video.video-sample-fps=$VIDEO_SAMPLE_FPS \
|
| 224 |
+
--data.video.video-size-pixels=$VIDEO_SIZE_PIXELS \
|
| 225 |
+
--data.video.video-decoder=$VIDEO_DECODER \
|
| 226 |
+
--data.video.video-cache-size=$VIDEO_CACHE_SIZE \
|
| 227 |
+
--data.video.sources-json=$PSI_VIDEO_SOURCES_JSON \
|
| 228 |
+
--data.video-clip-text-ratio=$PSI_VIDEO_CLIP_TEXT_TASK_WEIGHT \
|
| 229 |
+
--data.video-clip-only-ratio=$PSI_VIDEO_CLIP_ONLY_TASK_WEIGHT \
|
| 230 |
+
--data.video-clip.sources-json=$PSI_VIDEO_CLIP_SOURCES_JSON \
|
| 231 |
+
--data.video-clip.sample-stride=1 \
|
| 232 |
+
--data.video-clip.motion-fps=50 \
|
| 233 |
+
--data.video-clip.video-sample-fps=$VIDEO_CLIP_SAMPLE_FPS \
|
| 234 |
+
--data.video-clip.video-size-pixels=$VIDEO_SIZE_PIXELS \
|
| 235 |
+
--data.video-clip.video-decoder=$VIDEO_DECODER \
|
| 236 |
+
--data.video-clip.video-cache-size=$VIDEO_CACHE_SIZE \
|
| 237 |
+
--data.video-clip.action-exec-horizon=$ACTION_EXEC_HORIZON \
|
| 238 |
+
--data.sampler=batch_mixture \
|
| 239 |
+
--data.val-task=video_clip,video_clip_only \
|
| 240 |
+
--data.transform.repack.num-past-frames=$NUM_PAST_FRAMES \
|
| 241 |
+
--data.transform.repack.textop-actions \
|
| 242 |
+
--data.transform.repack.action-chunk-size=$ACTION_CHUNK_SIZE \
|
| 243 |
+
--data.transform.repack.pad-action-dim=39 \
|
| 244 |
+
--data.transform.repack.pad-state-dim=39 \
|
| 245 |
+
--data.transform.field.action_norm_type=bounds_q99 \
|
| 246 |
+
--data.transform.field.stat-path=assets/stats/seed_all_g1_motion39_textop_stats.json \
|
| 247 |
+
--data.transform.field.normalize-state \
|
| 248 |
+
--data.transform.field.pad-action-dim=39 \
|
| 249 |
+
--data.transform.field.pad-state-dim=39 \
|
| 250 |
+
--data.transform.model.resize.size 240 320 \
|
| 251 |
+
--data.transform.model.center_crop.size 240 320 \
|
| 252 |
+
--data.transform.model.no-img-aug \
|
| 253 |
+
--data.transform.model.text-only \
|
| 254 |
+
--data.transform.model.audio-only \
|
| 255 |
+
--data.transform.model.video-only \
|
| 256 |
+
--data.transform.model.video-size-pixels=$VIDEO_SIZE_PIXELS \
|
| 257 |
+
--model.model_name_or_path=$PSI_PRETRAIN_BACKBONE_PATH \
|
| 258 |
+
--model.noise-scheduler=flow \
|
| 259 |
+
--model.n_conditions=0 \
|
| 260 |
+
--model.action-chunk-size=$ACTION_CHUNK_SIZE \
|
| 261 |
+
--model.action-dim=39 \
|
| 262 |
+
--model.action-exec-horizon=$ACTION_EXEC_HORIZON \
|
| 263 |
+
--model.observation-horizon=$STATE_HORIZON \
|
| 264 |
+
--model.odim=39 \
|
| 265 |
+
--model.view_feature_dim=2048 \
|
| 266 |
+
--model.use-future-progress-embedding \
|
| 267 |
+
--model.future-progress-dim=$PROGRESS_DIM \
|
| 268 |
+
--model.tune-vlm \
|
| 269 |
+
--model.lang-backbone-lr=1e-5 \
|
| 270 |
+
--model.no-use_film \
|
| 271 |
+
--model.no-combined_temb \
|
| 272 |
+
--model.rtc \
|
| 273 |
+
--model.max-delay=11 \
|
| 274 |
+
--train.max-grad-value=1 \
|
| 275 |
+
--train.loss-group-weight.joint-abs=1.0 \
|
| 276 |
+
--train.loss-group-weight.root-delta-xy=3.0 \
|
| 277 |
+
--train.loss-group-weight.root-rot5-delta=10.0 \
|
| 278 |
+
--train.loss-group-weight.root-height=15.0 \
|
| 279 |
+
--train.loss-group-weight.foot-contact=0.1
|
| 280 |
+
"
|
| 281 |
+
torchrun --nproc_per_node=$NPROC_PER_NODE --master_port=29500 scripts/train.py \
|
| 282 |
+
$args "$@"
|
posttrain-v1-psi0-all.sh
ADDED
|
@@ -0,0 +1,281 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
pkill -f monitor.py
|
| 2 |
+
|
| 3 |
+
cd /cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/Psi0-yth-v18
|
| 4 |
+
export PYTHONPATH=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/Psi0-yth-v18/src
|
| 5 |
+
export CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7
|
| 6 |
+
export PSI_OUTPUT_DIR=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/Psi0-yth-v18/runs
|
| 7 |
+
export PSI_LOG_REPORT_TO=wandb
|
| 8 |
+
|
| 9 |
+
source .venv-psi/bin/activate
|
| 10 |
+
[ -f .env ] && source .env
|
| 11 |
+
|
| 12 |
+
export WANDB_API_KEY="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"
|
| 13 |
+
|
| 14 |
+
export OMP_NUM_THREADS=8
|
| 15 |
+
export TF_CPP_MIN_LOG_LEVEL=3
|
| 16 |
+
|
| 17 |
+
# Keep JSON sources readable in this script, but pass compact JSON to tyro.
|
| 18 |
+
compact_json() {
|
| 19 |
+
python -c 'import json, sys; print(json.dumps(json.load(sys.stdin), separators=(",", ":")))'
|
| 20 |
+
}
|
| 21 |
+
|
| 22 |
+
PSI_SEED_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/Psi0/dataset/seed_all
|
| 23 |
+
PSI_SEED_METADATA=metadata/seed_metadata_v004.csv
|
| 24 |
+
|
| 25 |
+
PSI_DANCE_MOTION_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/GR00T-WholeBodyControl/gear_sonic_deploy/reference/converted/psi0_dance/gvhmr
|
| 26 |
+
PSI_DANCE_AUDIO_VIDEO_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/dance_1723_video_psi0/dance_1723_video_10s
|
| 27 |
+
PSI_DANCE_VIDEO_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/dance_1723_video_psi0/dance_1723_video_10s
|
| 28 |
+
PSI_DANCE_VIDEO_EMBED_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/dance_1723_video_psi0/dance_1723_video_10s_embeds
|
| 29 |
+
PSI_VIDEO_DEMO_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/videogen/motion_example_0506_demo
|
| 30 |
+
PSI_QWEN3_TTS_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/qwen3-tts
|
| 31 |
+
PSI_VIDEO_CLIP_DATASET_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/videogen/psi0_video_clip_50f_rand8_full
|
| 32 |
+
PSI_DANCE_VIDEO_CLIP_ROOT=$PSI_VIDEO_CLIP_DATASET_ROOT/dance_1723_video_10s
|
| 33 |
+
PSI_MOTION_EXAMPLE_VIDEO_CLIP_ROOT=$PSI_VIDEO_CLIP_DATASET_ROOT/motion_example_0506_demo
|
| 34 |
+
PSI_MOTION_EXAMPLE_VIDEO_EMBED_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/videogen/motion_example_0506_demo_video_embeds
|
| 35 |
+
|
| 36 |
+
if [ -z "$PSI_AUDIO_SOURCES_JSON" ]; then
|
| 37 |
+
PSI_AUDIO_SOURCES_JSON=$(compact_json <<JSON
|
| 38 |
+
[
|
| 39 |
+
{
|
| 40 |
+
"name": "psi0_dance_audio",
|
| 41 |
+
"root_dir": "$PSI_DANCE_MOTION_ROOT",
|
| 42 |
+
"video_dir": "$PSI_DANCE_AUDIO_VIDEO_ROOT",
|
| 43 |
+
"dataset_name": "seed-audio-g1"
|
| 44 |
+
}
|
| 45 |
+
]
|
| 46 |
+
JSON
|
| 47 |
+
)
|
| 48 |
+
fi
|
| 49 |
+
|
| 50 |
+
if [ -z "$PSI_AUDIO_COMMAND_SOURCES_JSON" ]; then
|
| 51 |
+
PSI_AUDIO_COMMAND_SOURCES_JSON=$(compact_json <<JSON
|
| 52 |
+
[
|
| 53 |
+
{
|
| 54 |
+
"name": "qwen3_tts_seed_audio_command",
|
| 55 |
+
"root_dir": "$PSI_SEED_ROOT",
|
| 56 |
+
"dataset_root": "$PSI_QWEN3_TTS_ROOT",
|
| 57 |
+
"metadata_jsonl": "metadata.jsonl",
|
| 58 |
+
"motion_dir_key": "filename",
|
| 59 |
+
"audio_path_key": "audio_path",
|
| 60 |
+
"instruction_key": "content_short_description",
|
| 61 |
+
"motion_metadata_csv": "$PSI_SEED_METADATA",
|
| 62 |
+
"motion_metadata_key": "filename",
|
| 63 |
+
"motion_metadata_path_key": "move_g1_path",
|
| 64 |
+
"audio_file_extensions": ".wav",
|
| 65 |
+
"audio_task": "command",
|
| 66 |
+
"dataset_name": "seed-audio-command-g1"
|
| 67 |
+
}
|
| 68 |
+
]
|
| 69 |
+
JSON
|
| 70 |
+
)
|
| 71 |
+
fi
|
| 72 |
+
|
| 73 |
+
if [ -z "$PSI_VIDEO_SOURCES_JSON" ]; then
|
| 74 |
+
PSI_VIDEO_SOURCES_JSON=$(compact_json <<JSON
|
| 75 |
+
[
|
| 76 |
+
{
|
| 77 |
+
"name": "psi0_dance_gvhmr",
|
| 78 |
+
"root_dir": "$PSI_DANCE_MOTION_ROOT",
|
| 79 |
+
"video_dir": "$PSI_DANCE_VIDEO_ROOT",
|
| 80 |
+
"video_embed_dir": "$PSI_DANCE_VIDEO_EMBED_ROOT",
|
| 81 |
+
"dataset_name": "seed-video-g1"
|
| 82 |
+
},
|
| 83 |
+
{
|
| 84 |
+
"name": "motion_example_0506_demo",
|
| 85 |
+
"dataset_root": "$PSI_VIDEO_DEMO_ROOT",
|
| 86 |
+
"metadata_jsonl": "metadata.jsonl",
|
| 87 |
+
"motion_dir_key": "converted_path",
|
| 88 |
+
"video_path_key": "video_path",
|
| 89 |
+
"instruction_key": "video_caption",
|
| 90 |
+
"video_embed_dir": "$PSI_MOTION_EXAMPLE_VIDEO_EMBED_ROOT",
|
| 91 |
+
"dataset_name": "motion-example-video-g1"
|
| 92 |
+
}
|
| 93 |
+
]
|
| 94 |
+
JSON
|
| 95 |
+
)
|
| 96 |
+
fi
|
| 97 |
+
|
| 98 |
+
if [ -z "$PSI_VIDEO_CLIP_SOURCES_JSON" ]; then
|
| 99 |
+
PSI_VIDEO_CLIP_SOURCES_JSON=$(compact_json <<JSON
|
| 100 |
+
[
|
| 101 |
+
{
|
| 102 |
+
"name": "dance_1723_video_10s_clip50",
|
| 103 |
+
"dataset_root": "$PSI_DANCE_VIDEO_CLIP_ROOT",
|
| 104 |
+
"metadata_jsonl": "metadata.jsonl",
|
| 105 |
+
"motion_dir_key": "converted_path",
|
| 106 |
+
"video_path_key": "video_path",
|
| 107 |
+
"instruction_key": "video_caption",
|
| 108 |
+
"video_embed_dir": "$PSI_DANCE_VIDEO_CLIP_ROOT/embeds",
|
| 109 |
+
"dataset_name": "dance-video-clip-g1"
|
| 110 |
+
},
|
| 111 |
+
{
|
| 112 |
+
"name": "motion_example_0506_demo_clip50",
|
| 113 |
+
"dataset_root": "$PSI_MOTION_EXAMPLE_VIDEO_CLIP_ROOT",
|
| 114 |
+
"metadata_jsonl": "metadata.jsonl",
|
| 115 |
+
"motion_dir_key": "converted_path",
|
| 116 |
+
"video_path_key": "video_path",
|
| 117 |
+
"instruction_key": "video_caption",
|
| 118 |
+
"video_embed_dir": "$PSI_MOTION_EXAMPLE_VIDEO_CLIP_ROOT/embeds",
|
| 119 |
+
"dataset_name": "motion-example-video-clip-g1"
|
| 120 |
+
}
|
| 121 |
+
]
|
| 122 |
+
JSON
|
| 123 |
+
)
|
| 124 |
+
fi
|
| 125 |
+
|
| 126 |
+
PSI_TEXT_TASK_WEIGHT=1
|
| 127 |
+
PSI_AUDIO_TEXT_TASK_WEIGHT=0
|
| 128 |
+
PSI_AUDIO_ONLY_TASK_WEIGHT=0
|
| 129 |
+
PSI_AUDIO_COMMAND_TEXT_TASK_WEIGHT=0
|
| 130 |
+
PSI_AUDIO_COMMAND_ONLY_TASK_WEIGHT=0
|
| 131 |
+
PSI_VIDEO_TEXT_TASK_WEIGHT=0
|
| 132 |
+
PSI_VIDEO_ONLY_TASK_WEIGHT=0
|
| 133 |
+
PSI_VIDEO_CLIP_TEXT_TASK_WEIGHT=0
|
| 134 |
+
PSI_VIDEO_CLIP_ONLY_TASK_WEIGHT=0
|
| 135 |
+
|
| 136 |
+
PSI_PRETRAIN_BACKBONE_PATH=/cpfs01/VideoGen/public_ckpt/Qwen/Qwen2.5-Omni-3B
|
| 137 |
+
NUM_PAST_FRAMES=0
|
| 138 |
+
STATE_HORIZON=$((NUM_PAST_FRAMES + 1))
|
| 139 |
+
ACTION_CHUNK_SIZE=30
|
| 140 |
+
ACTION_EXEC_HORIZON=20
|
| 141 |
+
PROGRESS_DIM=5
|
| 142 |
+
VIDEO_SAMPLE_FPS=1.0
|
| 143 |
+
VIDEO_CLIP_SAMPLE_FPS=8.0
|
| 144 |
+
VIDEO_SIZE_PIXELS=50176
|
| 145 |
+
VIDEO_DECODER=ffmpeg
|
| 146 |
+
VIDEO_CACHE_SIZE=64
|
| 147 |
+
DATALOADER_NUM_WORKERS=4
|
| 148 |
+
VAL_DATALOADER_NUM_WORKERS=2
|
| 149 |
+
|
| 150 |
+
NPROC_PER_NODE=$(echo $CUDA_VISIBLE_DEVICES | tr ',' '\n' | wc -l)
|
| 151 |
+
|
| 152 |
+
echo "Training with $NPROC_PER_NODE GPU(s): $CUDA_VISIBLE_DEVICES"
|
| 153 |
+
echo "Text root: $PSI_SEED_ROOT"
|
| 154 |
+
echo "Dance motion root: $PSI_DANCE_MOTION_ROOT"
|
| 155 |
+
echo "Dance audio/video root: $PSI_DANCE_AUDIO_VIDEO_ROOT"
|
| 156 |
+
echo "Dance video root: $PSI_DANCE_VIDEO_ROOT"
|
| 157 |
+
echo "Dance video embed root: $PSI_DANCE_VIDEO_EMBED_ROOT"
|
| 158 |
+
echo "Qwen3-TTS acdudio command root: $PSI_QWEN3_TTS_ROOT"
|
| 159 |
+
echo "Audio sources: $PSI_AUDIO_SOURCES_JSON"
|
| 160 |
+
echo "Audio command sources: $PSI_AUDIO_COMMAND_SOURCES_JSON"
|
| 161 |
+
echo "Video sources: $PSI_VIDEO_SOURCES_JSON"
|
| 162 |
+
echo "Video clip sources: $PSI_VIDEO_CLIP_SOURCES_JSON"
|
| 163 |
+
echo "Task weights: text=$PSI_TEXT_TASK_WEIGHT audio_text=$PSI_AUDIO_TEXT_TASK_WEIGHT audio_only=$PSI_AUDIO_ONLY_TASK_WEIGHT audio_command_text=$PSI_AUDIO_COMMAND_TEXT_TASK_WEIGHT audio_command_only=$PSI_AUDIO_COMMAND_ONLY_TASK_WEIGHT video_text=$PSI_VIDEO_TEXT_TASK_WEIGHT video_only=$PSI_VIDEO_ONLY_TASK_WEIGHT video_clip_text=$PSI_VIDEO_CLIP_TEXT_TASK_WEIGHT video_clip_only=$PSI_VIDEO_CLIP_ONLY_TASK_WEIGHT"
|
| 164 |
+
# echo "Resume run dir: $PSI_PRETRAIN_RUN_DIR"
|
| 165 |
+
echo "TextOp 39D action/state enabled"
|
| 166 |
+
echo "Future progress embedding: enabled, dim $PROGRESS_DIM"
|
| 167 |
+
echo "Backbone path: $PSI_PRETRAIN_BACKBONE_PATH"
|
| 168 |
+
echo "Output dir: $PSI_OUTPUT_DIR"
|
| 169 |
+
|
| 170 |
+
args="posttrain_v1_seed_text_audio_video_mix_psi0_config \
|
| 171 |
+
--seed=292285 \
|
| 172 |
+
--exp=posttrain \
|
| 173 |
+
--timestamp=$(date +"%y%m%d%H%M") \
|
| 174 |
+
--train.name=posttrain_v1_trimodal \
|
| 175 |
+
--train.data_parallel=ddp \
|
| 176 |
+
--train.mixed_precision=bf16 \
|
| 177 |
+
--train.train_batch_size=64 \
|
| 178 |
+
--train.val_batch_size=64 \
|
| 179 |
+
--train.num-workers=$DATALOADER_NUM_WORKERS \
|
| 180 |
+
--train.val-num-workers=$VAL_DATALOADER_NUM_WORKERS \
|
| 181 |
+
--train.resume_from_checkpoint=None \
|
| 182 |
+
--train.max_checkpoints_to_keep=1 \
|
| 183 |
+
--train.gradient_accumulation_steps=1 \
|
| 184 |
+
--train.learning_rate=1e-4 \
|
| 185 |
+
--train.max_training_steps=1000000 \
|
| 186 |
+
--train.warmup_ratio=None \
|
| 187 |
+
--train.warmup_steps=1000 \
|
| 188 |
+
--train.checkpointing_steps=1000 \
|
| 189 |
+
--train.validation_steps=1000 \
|
| 190 |
+
--train.val_num_batches=20 \
|
| 191 |
+
--train.max_grad_norm=1.0 \
|
| 192 |
+
--train.lr_scheduler_type=cosine_with_min_lr \
|
| 193 |
+
--train.lr_scheduler_kwargs.betas 0.9 0.999 \
|
| 194 |
+
--train.lr_scheduler_kwargs.weight_decay=0 \
|
| 195 |
+
--train.lr_scheduler_kwargs.eps=1e-8 \
|
| 196 |
+
--log.report_to=$PSI_LOG_REPORT_TO \
|
| 197 |
+
--train.output_dir=$PSI_OUTPUT_DIR \
|
| 198 |
+
--data.text.root-dir=$PSI_SEED_ROOT \
|
| 199 |
+
--data.text.metadata-csv=$PSI_SEED_METADATA \
|
| 200 |
+
--data.text-ratio=$PSI_TEXT_TASK_WEIGHT \
|
| 201 |
+
--data.audio.sources-json=$PSI_AUDIO_SOURCES_JSON \
|
| 202 |
+
--data.audio-ratio=$PSI_AUDIO_TEXT_TASK_WEIGHT \
|
| 203 |
+
--data.audio-only-ratio=$PSI_AUDIO_ONLY_TASK_WEIGHT \
|
| 204 |
+
--data.audio.sample-stride=1 \
|
| 205 |
+
--data.audio.motion-fps=50 \
|
| 206 |
+
--data.audio.audio-sampling-rate=16000 \
|
| 207 |
+
--data.audio.audio-window-seconds=1.0 \
|
| 208 |
+
--data.audio.audio-left-context-seconds=0.0 \
|
| 209 |
+
--data.audio.full-audio \
|
| 210 |
+
--data.audio-command.sources-json=$PSI_AUDIO_COMMAND_SOURCES_JSON \
|
| 211 |
+
--data.audio-command-ratio=$PSI_AUDIO_COMMAND_TEXT_TASK_WEIGHT \
|
| 212 |
+
--data.audio-command-only-ratio=$PSI_AUDIO_COMMAND_ONLY_TASK_WEIGHT \
|
| 213 |
+
--data.audio-command.sample-stride=1 \
|
| 214 |
+
--data.audio-command.motion-fps=50 \
|
| 215 |
+
--data.audio-command.audio-sampling-rate=16000 \
|
| 216 |
+
--data.audio-command.audio-window-seconds=1.0 \
|
| 217 |
+
--data.audio-command.audio-left-context-seconds=0.0 \
|
| 218 |
+
--data.audio-command.full-audio \
|
| 219 |
+
--data.video-ratio=$PSI_VIDEO_TEXT_TASK_WEIGHT \
|
| 220 |
+
--data.video-only-ratio=$PSI_VIDEO_ONLY_TASK_WEIGHT \
|
| 221 |
+
--data.video.sample-stride=1 \
|
| 222 |
+
--data.video.motion-fps=50 \
|
| 223 |
+
--data.video.video-sample-fps=$VIDEO_SAMPLE_FPS \
|
| 224 |
+
--data.video.video-size-pixels=$VIDEO_SIZE_PIXELS \
|
| 225 |
+
--data.video.video-decoder=$VIDEO_DECODER \
|
| 226 |
+
--data.video.video-cache-size=$VIDEO_CACHE_SIZE \
|
| 227 |
+
--data.video.sources-json=$PSI_VIDEO_SOURCES_JSON \
|
| 228 |
+
--data.video-clip-text-ratio=$PSI_VIDEO_CLIP_TEXT_TASK_WEIGHT \
|
| 229 |
+
--data.video-clip-only-ratio=$PSI_VIDEO_CLIP_ONLY_TASK_WEIGHT \
|
| 230 |
+
--data.video-clip.sources-json=$PSI_VIDEO_CLIP_SOURCES_JSON \
|
| 231 |
+
--data.video-clip.sample-stride=1 \
|
| 232 |
+
--data.video-clip.motion-fps=50 \
|
| 233 |
+
--data.video-clip.video-sample-fps=$VIDEO_CLIP_SAMPLE_FPS \
|
| 234 |
+
--data.video-clip.video-size-pixels=$VIDEO_SIZE_PIXELS \
|
| 235 |
+
--data.video-clip.video-decoder=$VIDEO_DECODER \
|
| 236 |
+
--data.video-clip.video-cache-size=$VIDEO_CACHE_SIZE \
|
| 237 |
+
--data.video-clip.action-exec-horizon=$ACTION_EXEC_HORIZON \
|
| 238 |
+
--data.sampler=batch_mixture \
|
| 239 |
+
--data.val-task=text \
|
| 240 |
+
--data.transform.repack.num-past-frames=$NUM_PAST_FRAMES \
|
| 241 |
+
--data.transform.repack.textop-actions \
|
| 242 |
+
--data.transform.repack.action-chunk-size=$ACTION_CHUNK_SIZE \
|
| 243 |
+
--data.transform.repack.pad-action-dim=39 \
|
| 244 |
+
--data.transform.repack.pad-state-dim=39 \
|
| 245 |
+
--data.transform.field.action_norm_type=bounds_q99 \
|
| 246 |
+
--data.transform.field.stat-path=assets/stats/seed_all_g1_motion39_textop_stats.json \
|
| 247 |
+
--data.transform.field.normalize-state \
|
| 248 |
+
--data.transform.field.pad-action-dim=39 \
|
| 249 |
+
--data.transform.field.pad-state-dim=39 \
|
| 250 |
+
--data.transform.model.resize.size 240 320 \
|
| 251 |
+
--data.transform.model.center_crop.size 240 320 \
|
| 252 |
+
--data.transform.model.no-img-aug \
|
| 253 |
+
--data.transform.model.text-only \
|
| 254 |
+
--data.transform.model.audio-only \
|
| 255 |
+
--data.transform.model.video-only \
|
| 256 |
+
--data.transform.model.video-size-pixels=$VIDEO_SIZE_PIXELS \
|
| 257 |
+
--model.model_name_or_path=$PSI_PRETRAIN_BACKBONE_PATH \
|
| 258 |
+
--model.noise-scheduler=flow \
|
| 259 |
+
--model.n_conditions=0 \
|
| 260 |
+
--model.action-chunk-size=$ACTION_CHUNK_SIZE \
|
| 261 |
+
--model.action-dim=39 \
|
| 262 |
+
--model.action-exec-horizon=$ACTION_EXEC_HORIZON \
|
| 263 |
+
--model.observation-horizon=$STATE_HORIZON \
|
| 264 |
+
--model.odim=39 \
|
| 265 |
+
--model.view_feature_dim=2048 \
|
| 266 |
+
--model.use-future-progress-embedding \
|
| 267 |
+
--model.future-progress-dim=$PROGRESS_DIM \
|
| 268 |
+
--model.no-tune-vlm \
|
| 269 |
+
--model.no-use_film \
|
| 270 |
+
--model.no-combined_temb \
|
| 271 |
+
--model.rtc \
|
| 272 |
+
--model.max-delay=11 \
|
| 273 |
+
--train.max-grad-value=1 \
|
| 274 |
+
--train.loss-group-weight.joint-abs=1.0 \
|
| 275 |
+
--train.loss-group-weight.root-delta-xy=1.0 \
|
| 276 |
+
--train.loss-group-weight.root-rot5-delta=1.0 \
|
| 277 |
+
--train.loss-group-weight.root-height=1.0 \
|
| 278 |
+
--train.loss-group-weight.foot-contact=1.0
|
| 279 |
+
"
|
| 280 |
+
torchrun --nproc_per_node=$NPROC_PER_NODE --master_port=29500 scripts/train.py \
|
| 281 |
+
$args "$@"
|
posttrain-v1-psi0-all_resume.sh
ADDED
|
@@ -0,0 +1,285 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
pkill -f monitor.py
|
| 2 |
+
|
| 3 |
+
cd /cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/Psi0-yth-v18
|
| 4 |
+
export PYTHONPATH=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/Psi0-yth-v18/src
|
| 5 |
+
export CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7
|
| 6 |
+
export PSI_OUTPUT_DIR=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/Psi0-yth-v18/runs
|
| 7 |
+
export PSI_LOG_REPORT_TO=wandb
|
| 8 |
+
|
| 9 |
+
source .venv-psi/bin/activate
|
| 10 |
+
[ -f .env ] && source .env
|
| 11 |
+
|
| 12 |
+
export WANDB_API_KEY="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"
|
| 13 |
+
|
| 14 |
+
export OMP_NUM_THREADS=${OMP_NUM_THREADS:-1}
|
| 15 |
+
export TF_CPP_MIN_LOG_LEVEL=3
|
| 16 |
+
|
| 17 |
+
# Keep JSON sources readable in this script, but pass compact JSON to tyro.
|
| 18 |
+
compact_json() {
|
| 19 |
+
python -c 'import json, sys; print(json.dumps(json.load(sys.stdin), separators=(",", ":")))'
|
| 20 |
+
}
|
| 21 |
+
|
| 22 |
+
PSI_SEED_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/Psi0/dataset/seed_all
|
| 23 |
+
PSI_SEED_METADATA=metadata/seed_metadata_v004.csv
|
| 24 |
+
|
| 25 |
+
PSI_DANCE_MOTION_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/GR00T-WholeBodyControl/gear_sonic_deploy/reference/converted/psi0_dance/gvhmr
|
| 26 |
+
PSI_DANCE_AUDIO_VIDEO_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/dance_1723_video_psi0/dance_1723_video_10s
|
| 27 |
+
PSI_DANCE_VIDEO_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/dance_1723_video_psi0/dance_1723_video_10s
|
| 28 |
+
PSI_DANCE_VIDEO_EMBED_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/dance_1723_video_psi0/dance_1723_video_10s_embeds
|
| 29 |
+
PSI_VIDEO_DEMO_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/videogen/motion_example_0506_demo
|
| 30 |
+
PSI_QWEN3_TTS_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/qwen3-tts
|
| 31 |
+
PSI_VIDEO_CLIP_DATASET_ROOT=${PSI_VIDEO_CLIP_DATASET_ROOT:-/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/videogen/psi0_video_clip_50f_rand32_full}
|
| 32 |
+
PSI_DANCE_VIDEO_CLIP_ROOT=$PSI_VIDEO_CLIP_DATASET_ROOT/dance_1723_video_10s
|
| 33 |
+
PSI_MOTION_EXAMPLE_VIDEO_CLIP_ROOT=$PSI_VIDEO_CLIP_DATASET_ROOT/motion_example_0506_demo
|
| 34 |
+
PSI_MOTION_EXAMPLE_VIDEO_EMBED_ROOT=${PSI_MOTION_EXAMPLE_VIDEO_EMBED_ROOT:-/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/videogen/motion_example_0506_demo_video_embeds}
|
| 35 |
+
|
| 36 |
+
if [ -z "${PSI_AUDIO_SOURCES_JSON:-}" ]; then
|
| 37 |
+
PSI_AUDIO_SOURCES_JSON=$(compact_json <<JSON
|
| 38 |
+
[
|
| 39 |
+
{
|
| 40 |
+
"name": "psi0_dance_audio",
|
| 41 |
+
"root_dir": "$PSI_DANCE_MOTION_ROOT",
|
| 42 |
+
"video_dir": "$PSI_DANCE_AUDIO_VIDEO_ROOT",
|
| 43 |
+
"dataset_name": "seed-audio-g1"
|
| 44 |
+
}
|
| 45 |
+
]
|
| 46 |
+
JSON
|
| 47 |
+
)
|
| 48 |
+
fi
|
| 49 |
+
|
| 50 |
+
if [ -z "${PSI_AUDIO_COMMAND_SOURCES_JSON:-}" ]; then
|
| 51 |
+
PSI_AUDIO_COMMAND_SOURCES_JSON=$(compact_json <<JSON
|
| 52 |
+
[
|
| 53 |
+
{
|
| 54 |
+
"name": "qwen3_tts_seed_audio_command",
|
| 55 |
+
"root_dir": "$PSI_SEED_ROOT",
|
| 56 |
+
"dataset_root": "$PSI_QWEN3_TTS_ROOT",
|
| 57 |
+
"metadata_jsonl": "metadata.jsonl",
|
| 58 |
+
"motion_dir_key": "filename",
|
| 59 |
+
"audio_path_key": "audio_path",
|
| 60 |
+
"instruction_key": "content_short_description",
|
| 61 |
+
"motion_metadata_csv": "$PSI_SEED_METADATA",
|
| 62 |
+
"motion_metadata_key": "filename",
|
| 63 |
+
"motion_metadata_path_key": "move_g1_path",
|
| 64 |
+
"audio_file_extensions": ".wav",
|
| 65 |
+
"audio_task": "command",
|
| 66 |
+
"dataset_name": "seed-audio-command-g1"
|
| 67 |
+
}
|
| 68 |
+
]
|
| 69 |
+
JSON
|
| 70 |
+
)
|
| 71 |
+
fi
|
| 72 |
+
|
| 73 |
+
if [ -z "${PSI_VIDEO_SOURCES_JSON:-}" ]; then
|
| 74 |
+
PSI_VIDEO_SOURCES_JSON=$(compact_json <<JSON
|
| 75 |
+
[
|
| 76 |
+
{
|
| 77 |
+
"name": "psi0_dance_gvhmr",
|
| 78 |
+
"root_dir": "$PSI_DANCE_MOTION_ROOT",
|
| 79 |
+
"video_dir": "$PSI_DANCE_VIDEO_ROOT",
|
| 80 |
+
"video_embed_dir": "$PSI_DANCE_VIDEO_EMBED_ROOT",
|
| 81 |
+
"dataset_name": "seed-video-g1"
|
| 82 |
+
},
|
| 83 |
+
{
|
| 84 |
+
"name": "motion_example_0506_demo",
|
| 85 |
+
"dataset_root": "$PSI_VIDEO_DEMO_ROOT",
|
| 86 |
+
"metadata_jsonl": "metadata.jsonl",
|
| 87 |
+
"motion_dir_key": "converted_path",
|
| 88 |
+
"video_path_key": "video_path",
|
| 89 |
+
"instruction_key": "video_caption",
|
| 90 |
+
"video_embed_dir": "$PSI_MOTION_EXAMPLE_VIDEO_EMBED_ROOT",
|
| 91 |
+
"dataset_name": "motion-example-video-g1"
|
| 92 |
+
}
|
| 93 |
+
]
|
| 94 |
+
JSON
|
| 95 |
+
)
|
| 96 |
+
fi
|
| 97 |
+
|
| 98 |
+
|
| 99 |
+
|
| 100 |
+
|
| 101 |
+
if [ -z "${PSI_VIDEO_CLIP_SOURCES_JSON:-}" ]; then
|
| 102 |
+
PSI_VIDEO_CLIP_SOURCES_JSON=$(compact_json <<JSON
|
| 103 |
+
[
|
| 104 |
+
{
|
| 105 |
+
"name": "dance_1723_video_10s_clip50",
|
| 106 |
+
"dataset_root": "$PSI_DANCE_VIDEO_CLIP_ROOT",
|
| 107 |
+
"metadata_jsonl": "metadata.jsonl",
|
| 108 |
+
"motion_dir_key": "converted_path",
|
| 109 |
+
"video_path_key": "video_path",
|
| 110 |
+
"instruction_key": "video_caption",
|
| 111 |
+
"video_embed_dir": "$PSI_DANCE_VIDEO_CLIP_ROOT/embeds",
|
| 112 |
+
"dataset_name": "dance-video-clip-g1"
|
| 113 |
+
},
|
| 114 |
+
{
|
| 115 |
+
"name": "motion_example_0506_demo_clip50",
|
| 116 |
+
"dataset_root": "$PSI_MOTION_EXAMPLE_VIDEO_CLIP_ROOT",
|
| 117 |
+
"metadata_jsonl": "metadata.jsonl",
|
| 118 |
+
"motion_dir_key": "converted_path",
|
| 119 |
+
"video_path_key": "video_path",
|
| 120 |
+
"instruction_key": "video_caption",
|
| 121 |
+
"video_embed_dir": "$PSI_MOTION_EXAMPLE_VIDEO_CLIP_ROOT/embeds",
|
| 122 |
+
"dataset_name": "motion-example-video-clip-g1"
|
| 123 |
+
}
|
| 124 |
+
]
|
| 125 |
+
JSON
|
| 126 |
+
)
|
| 127 |
+
fi
|
| 128 |
+
|
| 129 |
+
|
| 130 |
+
|
| 131 |
+
PSI_TEXT_TASK_WEIGHT=1
|
| 132 |
+
PSI_AUDIO_TEXT_TASK_WEIGHT=0.5
|
| 133 |
+
PSI_AUDIO_ONLY_TASK_WEIGHT=0.5
|
| 134 |
+
PSI_AUDIO_COMMAND_TEXT_TASK_WEIGHT=0.5
|
| 135 |
+
PSI_AUDIO_COMMAND_ONLY_TASK_WEIGHT=0.5
|
| 136 |
+
PSI_VIDEO_TEXT_TASK_WEIGHT=0.5
|
| 137 |
+
PSI_VIDEO_ONLY_TASK_WEIGHT=0.5
|
| 138 |
+
PSI_VIDEO_CLIP_TEXT_TASK_WEIGHT=0.5
|
| 139 |
+
PSI_VIDEO_CLIP_ONLY_TASK_WEIGHT=0.5
|
| 140 |
+
|
| 141 |
+
PSI_PRETRAIN_BACKBONE_PATH=/cpfs01/VideoGen/public_ckpt/Qwen/Qwen2.5-Omni-3B
|
| 142 |
+
# PSI_PRETRAIN_RUN_DIR=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/Psi0-yth-v9/runs/posttrain_v1/posttrain.sg.flow1000.cwml.lr1.0e-04.b512.gpus8.2604250426
|
| 143 |
+
PSI_PRETRAIN_RUN_DIR=${PSI_PRETRAIN_RUN_DIR:-/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/Psi0-yth-v12/runs/posttrain_v1_trimodal/posttrain.sg.flow1000.cwml.lr1.0e-04.b512.gpus8.2605140518}
|
| 144 |
+
|
| 145 |
+
NUM_PAST_FRAMES=0
|
| 146 |
+
STATE_HORIZON=$((NUM_PAST_FRAMES + 1))
|
| 147 |
+
ACTION_CHUNK_SIZE=30
|
| 148 |
+
ACTION_EXEC_HORIZON=20
|
| 149 |
+
VIDEO_SAMPLE_FPS=1.0
|
| 150 |
+
VIDEO_CLIP_SAMPLE_FPS=${VIDEO_CLIP_SAMPLE_FPS:-32.0}
|
| 151 |
+
VIDEO_SIZE_PIXELS=50176
|
| 152 |
+
VIDEO_DECODER=${VIDEO_DECODER:-ffmpeg}
|
| 153 |
+
VIDEO_CACHE_SIZE=${VIDEO_CACHE_SIZE:-64}
|
| 154 |
+
DATALOADER_NUM_WORKERS=${DATALOADER_NUM_WORKERS:-4}
|
| 155 |
+
VAL_DATALOADER_NUM_WORKERS=${VAL_DATALOADER_NUM_WORKERS:-2}
|
| 156 |
+
|
| 157 |
+
NPROC_PER_NODE=$(echo $CUDA_VISIBLE_DEVICES | tr ',' '\n' | wc -l)
|
| 158 |
+
|
| 159 |
+
echo "Training with $NPROC_PER_NODE GPU(s): $CUDA_VISIBLE_DEVICES"
|
| 160 |
+
echo "Text root: $PSI_SEED_ROOT"
|
| 161 |
+
echo "Dance motion root: $PSI_DANCE_MOTION_ROOT"
|
| 162 |
+
echo "Dance audio/video root: $PSI_DANCE_AUDIO_VIDEO_ROOT"
|
| 163 |
+
echo "Dance video root: $PSI_DANCE_VIDEO_ROOT"
|
| 164 |
+
echo "Dance video embed root: $PSI_DANCE_VIDEO_EMBED_ROOT"
|
| 165 |
+
echo "Qwen3-TTS audio command root: $PSI_QWEN3_TTS_ROOT"
|
| 166 |
+
echo "Audio sources: $PSI_AUDIO_SOURCES_JSON"
|
| 167 |
+
echo "Audio command sources: $PSI_AUDIO_COMMAND_SOURCES_JSON"
|
| 168 |
+
echo "Video sources: $PSI_VIDEO_SOURCES_JSON"
|
| 169 |
+
echo "Video clip sources: $PSI_VIDEO_CLIP_SOURCES_JSON"
|
| 170 |
+
echo "Task weights: text=$PSI_TEXT_TASK_WEIGHT audio_text=$PSI_AUDIO_TEXT_TASK_WEIGHT audio_only=$PSI_AUDIO_ONLY_TASK_WEIGHT audio_command_text=$PSI_AUDIO_COMMAND_TEXT_TASK_WEIGHT audio_command_only=$PSI_AUDIO_COMMAND_ONLY_TASK_WEIGHT video_text=$PSI_VIDEO_TEXT_TASK_WEIGHT video_only=$PSI_VIDEO_ONLY_TASK_WEIGHT video_clip_text=$PSI_VIDEO_CLIP_TEXT_TASK_WEIGHT video_clip_only=$PSI_VIDEO_CLIP_ONLY_TASK_WEIGHT"
|
| 171 |
+
echo "Resume run dir: $PSI_PRETRAIN_RUN_DIR"
|
| 172 |
+
echo "Backbone path: $PSI_PRETRAIN_BACKBONE_PATH"
|
| 173 |
+
echo "Output dir: $PSI_OUTPUT_DIR"
|
| 174 |
+
|
| 175 |
+
args="posttrain_v1_seed_text_audio_video_mix_psi0_config \
|
| 176 |
+
--seed=292285 \
|
| 177 |
+
--exp=posttrain \
|
| 178 |
+
--timestamp=$(date +"%y%m%d%H%M") \
|
| 179 |
+
--train.name=posttrain_v1_trimodal \
|
| 180 |
+
--train.data_parallel=ddp \
|
| 181 |
+
--train.mixed_precision=bf16 \
|
| 182 |
+
--train.train_batch_size=64 \
|
| 183 |
+
--train.val_batch_size=64 \
|
| 184 |
+
--train.num-workers=$DATALOADER_NUM_WORKERS \
|
| 185 |
+
--train.val-num-workers=$VAL_DATALOADER_NUM_WORKERS \
|
| 186 |
+
--train.resume_from_checkpoint=$PSI_PRETRAIN_RUN_DIR \
|
| 187 |
+
--train.max_checkpoints_to_keep=1 \
|
| 188 |
+
--train.gradient_accumulation_steps=1 \
|
| 189 |
+
--train.learning_rate=1e-4 \
|
| 190 |
+
--train.max_training_steps=1000000 \
|
| 191 |
+
--train.warmup_ratio=None \
|
| 192 |
+
--train.warmup_steps=1000 \
|
| 193 |
+
--train.checkpointing_steps=1000 \
|
| 194 |
+
--train.validation_steps=1000 \
|
| 195 |
+
--train.val_num_batches=20 \
|
| 196 |
+
--train.max_grad_norm=1.0 \
|
| 197 |
+
--train.lr_scheduler_type=cosine_with_min_lr \
|
| 198 |
+
--train.lr_scheduler_kwargs.betas 0.9 0.999 \
|
| 199 |
+
--train.lr_scheduler_kwargs.weight_decay=0 \
|
| 200 |
+
--train.lr_scheduler_kwargs.eps=1e-8 \
|
| 201 |
+
--log.report_to=$PSI_LOG_REPORT_TO \
|
| 202 |
+
--train.output_dir=$PSI_OUTPUT_DIR \
|
| 203 |
+
--data.text.root-dir=$PSI_SEED_ROOT \
|
| 204 |
+
--data.text.metadata-csv=$PSI_SEED_METADATA \
|
| 205 |
+
--data.text-ratio=$PSI_TEXT_TASK_WEIGHT \
|
| 206 |
+
--data.audio.sources-json=$PSI_AUDIO_SOURCES_JSON \
|
| 207 |
+
--data.audio-ratio=$PSI_AUDIO_TEXT_TASK_WEIGHT \
|
| 208 |
+
--data.audio-only-ratio=$PSI_AUDIO_ONLY_TASK_WEIGHT \
|
| 209 |
+
--data.audio.sample-stride=1 \
|
| 210 |
+
--data.audio.motion-fps=50 \
|
| 211 |
+
--data.audio.audio-sampling-rate=16000 \
|
| 212 |
+
--data.audio.audio-window-seconds=1.0 \
|
| 213 |
+
--data.audio.audio-left-context-seconds=0.0 \
|
| 214 |
+
--data.audio.full-audio \
|
| 215 |
+
--data.audio-command.sources-json=$PSI_AUDIO_COMMAND_SOURCES_JSON \
|
| 216 |
+
--data.audio-command-ratio=$PSI_AUDIO_COMMAND_TEXT_TASK_WEIGHT \
|
| 217 |
+
--data.audio-command-only-ratio=$PSI_AUDIO_COMMAND_ONLY_TASK_WEIGHT \
|
| 218 |
+
--data.audio-command.sample-stride=1 \
|
| 219 |
+
--data.audio-command.motion-fps=50 \
|
| 220 |
+
--data.audio-command.audio-sampling-rate=16000 \
|
| 221 |
+
--data.audio-command.audio-window-seconds=1.0 \
|
| 222 |
+
--data.audio-command.audio-left-context-seconds=0.0 \
|
| 223 |
+
--data.audio-command.full-audio \
|
| 224 |
+
--data.video-ratio=$PSI_VIDEO_TEXT_TASK_WEIGHT \
|
| 225 |
+
--data.video-only-ratio=$PSI_VIDEO_ONLY_TASK_WEIGHT \
|
| 226 |
+
--data.video.sample-stride=1 \
|
| 227 |
+
--data.video.motion-fps=50 \
|
| 228 |
+
--data.video.video-sample-fps=$VIDEO_SAMPLE_FPS \
|
| 229 |
+
--data.video.video-size-pixels=$VIDEO_SIZE_PIXELS \
|
| 230 |
+
--data.video.video-decoder=$VIDEO_DECODER \
|
| 231 |
+
--data.video.video-cache-size=$VIDEO_CACHE_SIZE \
|
| 232 |
+
--data.video.sources-json=$PSI_VIDEO_SOURCES_JSON \
|
| 233 |
+
--data.video-clip-text-ratio=$PSI_VIDEO_CLIP_TEXT_TASK_WEIGHT \
|
| 234 |
+
--data.video-clip-only-ratio=$PSI_VIDEO_CLIP_ONLY_TASK_WEIGHT \
|
| 235 |
+
--data.video-clip.sources-json=$PSI_VIDEO_CLIP_SOURCES_JSON \
|
| 236 |
+
--data.video-clip.sample-stride=1 \
|
| 237 |
+
--data.video-clip.motion-fps=50 \
|
| 238 |
+
--data.video-clip.video-sample-fps=$VIDEO_CLIP_SAMPLE_FPS \
|
| 239 |
+
--data.video-clip.video-size-pixels=$VIDEO_SIZE_PIXELS \
|
| 240 |
+
--data.video-clip.video-decoder=$VIDEO_DECODER \
|
| 241 |
+
--data.video-clip.video-cache-size=$VIDEO_CACHE_SIZE \
|
| 242 |
+
--data.video-clip.action-exec-horizon=$ACTION_EXEC_HORIZON \
|
| 243 |
+
--data.sampler=batch_mixture \
|
| 244 |
+
--data.val-task=text,audio,audio_only,audio_command,audio_command_only,video,video_only,video_clip,video_clip_only \
|
| 245 |
+
--data.transform.repack.num-past-frames=$NUM_PAST_FRAMES \
|
| 246 |
+
--data.transform.repack.textop-actions \
|
| 247 |
+
--data.transform.repack.action-chunk-size=$ACTION_CHUNK_SIZE \
|
| 248 |
+
--data.transform.repack.pad-action-dim=70 \
|
| 249 |
+
--data.transform.repack.pad-state-dim=36 \
|
| 250 |
+
--data.transform.field.action_norm_type=bounds_q99 \
|
| 251 |
+
--data.transform.field.stat-path=assets/stats/seed_all_g1_motion70_textop_stats.json \
|
| 252 |
+
--data.transform.field.normalize-state \
|
| 253 |
+
--data.transform.field.pad-action-dim=70 \
|
| 254 |
+
--data.transform.field.pad-state-dim=36 \
|
| 255 |
+
--data.transform.model.resize.size 240 320 \
|
| 256 |
+
--data.transform.model.center_crop.size 240 320 \
|
| 257 |
+
--data.transform.model.no-img-aug \
|
| 258 |
+
--data.transform.model.text-only \
|
| 259 |
+
--data.transform.model.audio-only \
|
| 260 |
+
--data.transform.model.video-only \
|
| 261 |
+
--data.transform.model.video-size-pixels=$VIDEO_SIZE_PIXELS \
|
| 262 |
+
--model.model_name_or_path=$PSI_PRETRAIN_BACKBONE_PATH \
|
| 263 |
+
--model.noise-scheduler=flow \
|
| 264 |
+
--model.n_conditions=0 \
|
| 265 |
+
--model.action-chunk-size=$ACTION_CHUNK_SIZE \
|
| 266 |
+
--model.action-dim=70 \
|
| 267 |
+
--model.action-exec-horizon=$ACTION_EXEC_HORIZON \
|
| 268 |
+
--model.observation-horizon=$STATE_HORIZON \
|
| 269 |
+
--model.odim=36 \
|
| 270 |
+
--model.view_feature_dim=2048 \
|
| 271 |
+
--model.no-tune-vlm \
|
| 272 |
+
--model.no-use_film \
|
| 273 |
+
--model.no-combined_temb \
|
| 274 |
+
--model.rtc \
|
| 275 |
+
--model.max-delay=11 \
|
| 276 |
+
--train.max-grad-value=1 \
|
| 277 |
+
--train.loss-group-weight.joint-abs=0.1 \
|
| 278 |
+
--train.loss-group-weight.joint-delta=1.0 \
|
| 279 |
+
--train.loss-group-weight.root-delta-local=1.0 \
|
| 280 |
+
--train.loss-group-weight.root-rot6d-delta=1.0 \
|
| 281 |
+
--train.loss-group-weight.root-height=1.0 \
|
| 282 |
+
--train.loss-group-weight.foot-contact=1.0
|
| 283 |
+
"
|
| 284 |
+
torchrun --nproc_per_node=$NPROC_PER_NODE --master_port=29500 scripts/train.py \
|
| 285 |
+
${args} "$@"
|
posttrain-v1-psi0-audio-clip-tune-vlm.sh
ADDED
|
@@ -0,0 +1,282 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
pkill -f monitor.py
|
| 2 |
+
|
| 3 |
+
cd /cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/Psi0-yth-v18
|
| 4 |
+
export PYTHONPATH=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/Psi0-yth-v18/src
|
| 5 |
+
export CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7
|
| 6 |
+
export PSI_OUTPUT_DIR=${PSI_OUTPUT_DIR:-/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/Psi0-yth-v18/runs}
|
| 7 |
+
export PSI_LOG_REPORT_TO=wandb
|
| 8 |
+
|
| 9 |
+
source /cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/Psi0-yth-v15/.venv-psi/bin/activate
|
| 10 |
+
[ -f .env ] && source .env
|
| 11 |
+
|
| 12 |
+
export WANDB_API_KEY="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"
|
| 13 |
+
|
| 14 |
+
export OMP_NUM_THREADS=8
|
| 15 |
+
export TF_CPP_MIN_LOG_LEVEL=3
|
| 16 |
+
|
| 17 |
+
# Keep JSON sources readable in this script, but pass compact JSON to tyro.
|
| 18 |
+
compact_json() {
|
| 19 |
+
python -c 'import json, sys; print(json.dumps(json.load(sys.stdin), separators=(",", ":")))'
|
| 20 |
+
}
|
| 21 |
+
|
| 22 |
+
PSI_SEED_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/Psi0/dataset/seed_all
|
| 23 |
+
PSI_SEED_METADATA=metadata/seed_metadata_v004.csv
|
| 24 |
+
|
| 25 |
+
PSI_DANCE_MOTION_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/GR00T-WholeBodyControl/gear_sonic_deploy/reference/converted/psi0_dance/gvhmr
|
| 26 |
+
PSI_DANCE_AUDIO_VIDEO_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/dance_1723_video_psi0/dance_1723_video_10s
|
| 27 |
+
PSI_DANCE_VIDEO_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/dance_1723_video_psi0/dance_1723_video_10s
|
| 28 |
+
PSI_DANCE_VIDEO_EMBED_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/dance_1723_video_psi0/dance_1723_video_10s_embeds
|
| 29 |
+
PSI_VIDEO_DEMO_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/videogen/motion_example_0506_demo
|
| 30 |
+
PSI_QWEN3_TTS_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/qwen3-tts
|
| 31 |
+
PSI_VIDEO_CLIP_DATASET_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/videogen/psi0_video_clip_50f_rand8_full
|
| 32 |
+
PSI_DANCE_VIDEO_CLIP_ROOT=$PSI_VIDEO_CLIP_DATASET_ROOT/dance_1723_video_10s
|
| 33 |
+
PSI_MOTION_EXAMPLE_VIDEO_CLIP_ROOT=$PSI_VIDEO_CLIP_DATASET_ROOT/motion_example_0506_demo
|
| 34 |
+
PSI_MOTION_EXAMPLE_VIDEO_EMBED_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/videogen/motion_example_0506_demo_video_embeds
|
| 35 |
+
|
| 36 |
+
if [ -z "$PSI_AUDIO_SOURCES_JSON" ]; then
|
| 37 |
+
PSI_AUDIO_SOURCES_JSON=$(compact_json <<JSON
|
| 38 |
+
[
|
| 39 |
+
{
|
| 40 |
+
"name": "psi0_dance_audio",
|
| 41 |
+
"root_dir": "$PSI_DANCE_MOTION_ROOT",
|
| 42 |
+
"video_dir": "$PSI_DANCE_AUDIO_VIDEO_ROOT",
|
| 43 |
+
"dataset_name": "seed-audio-g1"
|
| 44 |
+
}
|
| 45 |
+
]
|
| 46 |
+
JSON
|
| 47 |
+
)
|
| 48 |
+
fi
|
| 49 |
+
|
| 50 |
+
if [ -z "$PSI_AUDIO_COMMAND_SOURCES_JSON" ]; then
|
| 51 |
+
PSI_AUDIO_COMMAND_SOURCES_JSON=$(compact_json <<JSON
|
| 52 |
+
[
|
| 53 |
+
{
|
| 54 |
+
"name": "qwen3_tts_seed_audio_command",
|
| 55 |
+
"root_dir": "$PSI_SEED_ROOT",
|
| 56 |
+
"dataset_root": "$PSI_QWEN3_TTS_ROOT",
|
| 57 |
+
"metadata_jsonl": "metadata.jsonl",
|
| 58 |
+
"motion_dir_key": "filename",
|
| 59 |
+
"audio_path_key": "audio_path",
|
| 60 |
+
"instruction_key": "content_short_description",
|
| 61 |
+
"motion_metadata_csv": "$PSI_SEED_METADATA",
|
| 62 |
+
"motion_metadata_key": "filename",
|
| 63 |
+
"motion_metadata_path_key": "move_g1_path",
|
| 64 |
+
"audio_file_extensions": ".wav",
|
| 65 |
+
"audio_task": "command",
|
| 66 |
+
"dataset_name": "seed-audio-command-g1"
|
| 67 |
+
}
|
| 68 |
+
]
|
| 69 |
+
JSON
|
| 70 |
+
)
|
| 71 |
+
fi
|
| 72 |
+
|
| 73 |
+
if [ -z "$PSI_VIDEO_SOURCES_JSON" ]; then
|
| 74 |
+
PSI_VIDEO_SOURCES_JSON=$(compact_json <<JSON
|
| 75 |
+
[
|
| 76 |
+
{
|
| 77 |
+
"name": "psi0_dance_gvhmr",
|
| 78 |
+
"root_dir": "$PSI_DANCE_MOTION_ROOT",
|
| 79 |
+
"video_dir": "$PSI_DANCE_VIDEO_ROOT",
|
| 80 |
+
"video_embed_dir": "$PSI_DANCE_VIDEO_EMBED_ROOT",
|
| 81 |
+
"dataset_name": "seed-video-g1"
|
| 82 |
+
},
|
| 83 |
+
{
|
| 84 |
+
"name": "motion_example_0506_demo",
|
| 85 |
+
"dataset_root": "$PSI_VIDEO_DEMO_ROOT",
|
| 86 |
+
"metadata_jsonl": "metadata.jsonl",
|
| 87 |
+
"motion_dir_key": "converted_path",
|
| 88 |
+
"video_path_key": "video_path",
|
| 89 |
+
"instruction_key": "video_caption",
|
| 90 |
+
"video_embed_dir": "$PSI_MOTION_EXAMPLE_VIDEO_EMBED_ROOT",
|
| 91 |
+
"dataset_name": "motion-example-video-g1"
|
| 92 |
+
}
|
| 93 |
+
]
|
| 94 |
+
JSON
|
| 95 |
+
)
|
| 96 |
+
fi
|
| 97 |
+
|
| 98 |
+
if [ -z "$PSI_VIDEO_CLIP_SOURCES_JSON" ]; then
|
| 99 |
+
PSI_VIDEO_CLIP_SOURCES_JSON=$(compact_json <<JSON
|
| 100 |
+
[
|
| 101 |
+
{
|
| 102 |
+
"name": "dance_1723_video_10s_clip50",
|
| 103 |
+
"dataset_root": "$PSI_DANCE_VIDEO_CLIP_ROOT",
|
| 104 |
+
"metadata_jsonl": "metadata.jsonl",
|
| 105 |
+
"motion_dir_key": "converted_path",
|
| 106 |
+
"video_path_key": "video_path",
|
| 107 |
+
"instruction_key": "video_caption",
|
| 108 |
+
"video_embed_dir": "$PSI_DANCE_VIDEO_CLIP_ROOT/embeds",
|
| 109 |
+
"dataset_name": "dance-video-clip-g1"
|
| 110 |
+
},
|
| 111 |
+
{
|
| 112 |
+
"name": "motion_example_0506_demo_clip50",
|
| 113 |
+
"dataset_root": "$PSI_MOTION_EXAMPLE_VIDEO_CLIP_ROOT",
|
| 114 |
+
"metadata_jsonl": "metadata.jsonl",
|
| 115 |
+
"motion_dir_key": "converted_path",
|
| 116 |
+
"video_path_key": "video_path",
|
| 117 |
+
"instruction_key": "video_caption",
|
| 118 |
+
"video_embed_dir": "$PSI_MOTION_EXAMPLE_VIDEO_CLIP_ROOT/embeds",
|
| 119 |
+
"dataset_name": "motion-example-video-clip-g1"
|
| 120 |
+
}
|
| 121 |
+
]
|
| 122 |
+
JSON
|
| 123 |
+
)
|
| 124 |
+
fi
|
| 125 |
+
|
| 126 |
+
PSI_TEXT_TASK_WEIGHT=0
|
| 127 |
+
PSI_AUDIO_TEXT_TASK_WEIGHT=0.5
|
| 128 |
+
PSI_AUDIO_ONLY_TASK_WEIGHT=0.5
|
| 129 |
+
PSI_AUDIO_COMMAND_TEXT_TASK_WEIGHT=0
|
| 130 |
+
PSI_AUDIO_COMMAND_ONLY_TASK_WEIGHT=0
|
| 131 |
+
PSI_VIDEO_TEXT_TASK_WEIGHT=0
|
| 132 |
+
PSI_VIDEO_ONLY_TASK_WEIGHT=0
|
| 133 |
+
PSI_VIDEO_CLIP_TEXT_TASK_WEIGHT=0
|
| 134 |
+
PSI_VIDEO_CLIP_ONLY_TASK_WEIGHT=0
|
| 135 |
+
|
| 136 |
+
PSI_PRETRAIN_BACKBONE_PATH=/cpfs01/VideoGen/public_ckpt/Qwen/Qwen2.5-Omni-3B
|
| 137 |
+
NUM_PAST_FRAMES=0
|
| 138 |
+
STATE_HORIZON=$((NUM_PAST_FRAMES + 1))
|
| 139 |
+
ACTION_CHUNK_SIZE=30
|
| 140 |
+
ACTION_EXEC_HORIZON=20
|
| 141 |
+
PROGRESS_DIM=5
|
| 142 |
+
VIDEO_SAMPLE_FPS=1.0
|
| 143 |
+
VIDEO_CLIP_SAMPLE_FPS=8.0
|
| 144 |
+
VIDEO_SIZE_PIXELS=50176
|
| 145 |
+
VIDEO_DECODER=ffmpeg
|
| 146 |
+
VIDEO_CACHE_SIZE=64
|
| 147 |
+
DATALOADER_NUM_WORKERS=4
|
| 148 |
+
VAL_DATALOADER_NUM_WORKERS=2
|
| 149 |
+
|
| 150 |
+
NPROC_PER_NODE=$(echo $CUDA_VISIBLE_DEVICES | tr ',' '\n' | wc -l)
|
| 151 |
+
|
| 152 |
+
echo "Training with $NPROC_PER_NODE GPU(s): $CUDA_VISIBLE_DEVICES"
|
| 153 |
+
echo "Text root: $PSI_SEED_ROOT"
|
| 154 |
+
echo "Dance motion root: $PSI_DANCE_MOTION_ROOT"
|
| 155 |
+
echo "Dance audio/video root: $PSI_DANCE_AUDIO_VIDEO_ROOT"
|
| 156 |
+
echo "Dance video root: $PSI_DANCE_VIDEO_ROOT"
|
| 157 |
+
echo "Dance video embed root: $PSI_DANCE_VIDEO_EMBED_ROOT"
|
| 158 |
+
echo "Qwen3-TTS acdudio command root: $PSI_QWEN3_TTS_ROOT"
|
| 159 |
+
echo "Audio sources: $PSI_AUDIO_SOURCES_JSON"
|
| 160 |
+
echo "Audio command sources: $PSI_AUDIO_COMMAND_SOURCES_JSON"
|
| 161 |
+
echo "Video sources: $PSI_VIDEO_SOURCES_JSON"
|
| 162 |
+
echo "Video clip sources: $PSI_VIDEO_CLIP_SOURCES_JSON"
|
| 163 |
+
echo "Task weights: text=$PSI_TEXT_TASK_WEIGHT audio_text=$PSI_AUDIO_TEXT_TASK_WEIGHT audio_only=$PSI_AUDIO_ONLY_TASK_WEIGHT audio_command_text=$PSI_AUDIO_COMMAND_TEXT_TASK_WEIGHT audio_command_only=$PSI_AUDIO_COMMAND_ONLY_TASK_WEIGHT video_text=$PSI_VIDEO_TEXT_TASK_WEIGHT video_only=$PSI_VIDEO_ONLY_TASK_WEIGHT video_clip_text=$PSI_VIDEO_CLIP_TEXT_TASK_WEIGHT video_clip_only=$PSI_VIDEO_CLIP_ONLY_TASK_WEIGHT"
|
| 164 |
+
# echo "Resume run dir: $PSI_PRETRAIN_RUN_DIR"
|
| 165 |
+
echo "TextOp 39D action/state enabled"
|
| 166 |
+
echo "Future progress embedding: enabled, dim $PROGRESS_DIM"
|
| 167 |
+
echo "Backbone path: $PSI_PRETRAIN_BACKBONE_PATH"
|
| 168 |
+
echo "Output dir: $PSI_OUTPUT_DIR"
|
| 169 |
+
|
| 170 |
+
args="posttrain_v1_seed_text_audio_video_mix_psi0_config \
|
| 171 |
+
--seed=292285 \
|
| 172 |
+
--exp=posttrain \
|
| 173 |
+
--timestamp=$(date +"%y%m%d%H%M") \
|
| 174 |
+
--train.name=posttrain_v1_trimodal \
|
| 175 |
+
--train.data_parallel=ddp \
|
| 176 |
+
--train.mixed_precision=bf16 \
|
| 177 |
+
--train.train_batch_size=32 \
|
| 178 |
+
--train.val_batch_size=64 \
|
| 179 |
+
--train.num-workers=$DATALOADER_NUM_WORKERS \
|
| 180 |
+
--train.val-num-workers=$VAL_DATALOADER_NUM_WORKERS \
|
| 181 |
+
--train.resume_from_checkpoint=None \
|
| 182 |
+
--train.max_checkpoints_to_keep=1 \
|
| 183 |
+
--train.gradient_accumulation_steps=1 \
|
| 184 |
+
--train.learning_rate=1e-4 \
|
| 185 |
+
--train.max_training_steps=500000 \
|
| 186 |
+
--train.warmup_ratio=None \
|
| 187 |
+
--train.warmup_steps=1000 \
|
| 188 |
+
--train.checkpointing_steps=1000 \
|
| 189 |
+
--train.validation_steps=1000 \
|
| 190 |
+
--train.val_num_batches=20 \
|
| 191 |
+
--train.max_grad_norm=1.0 \
|
| 192 |
+
--train.lr_scheduler_type=cosine_with_min_lr \
|
| 193 |
+
--train.lr_scheduler_kwargs.betas 0.9 0.999 \
|
| 194 |
+
--train.lr_scheduler_kwargs.weight_decay=0 \
|
| 195 |
+
--train.lr_scheduler_kwargs.eps=1e-8 \
|
| 196 |
+
--log.report_to=$PSI_LOG_REPORT_TO \
|
| 197 |
+
--train.output_dir=$PSI_OUTPUT_DIR \
|
| 198 |
+
--data.text.root-dir=$PSI_SEED_ROOT \
|
| 199 |
+
--data.text.metadata-csv=$PSI_SEED_METADATA \
|
| 200 |
+
--data.text-ratio=$PSI_TEXT_TASK_WEIGHT \
|
| 201 |
+
--data.audio.sources-json=$PSI_AUDIO_SOURCES_JSON \
|
| 202 |
+
--data.audio-ratio=$PSI_AUDIO_TEXT_TASK_WEIGHT \
|
| 203 |
+
--data.audio-only-ratio=$PSI_AUDIO_ONLY_TASK_WEIGHT \
|
| 204 |
+
--data.audio.sample-stride=1 \
|
| 205 |
+
--data.audio.motion-fps=50 \
|
| 206 |
+
--data.audio.audio-sampling-rate=16000 \
|
| 207 |
+
--data.audio.audio-window-seconds=0.6 \
|
| 208 |
+
--data.audio.audio-left-context-seconds=0.0 \
|
| 209 |
+
--data.audio.no-full-audio \
|
| 210 |
+
--data.audio-command.sources-json=$PSI_AUDIO_COMMAND_SOURCES_JSON \
|
| 211 |
+
--data.audio-command-ratio=$PSI_AUDIO_COMMAND_TEXT_TASK_WEIGHT \
|
| 212 |
+
--data.audio-command-only-ratio=$PSI_AUDIO_COMMAND_ONLY_TASK_WEIGHT \
|
| 213 |
+
--data.audio-command.sample-stride=1 \
|
| 214 |
+
--data.audio-command.motion-fps=50 \
|
| 215 |
+
--data.audio-command.audio-sampling-rate=16000 \
|
| 216 |
+
--data.audio-command.audio-window-seconds=1.0 \
|
| 217 |
+
--data.audio-command.audio-left-context-seconds=0.0 \
|
| 218 |
+
--data.audio-command.full-audio \
|
| 219 |
+
--data.video-ratio=$PSI_VIDEO_TEXT_TASK_WEIGHT \
|
| 220 |
+
--data.video-only-ratio=$PSI_VIDEO_ONLY_TASK_WEIGHT \
|
| 221 |
+
--data.video.sample-stride=1 \
|
| 222 |
+
--data.video.motion-fps=50 \
|
| 223 |
+
--data.video.video-sample-fps=$VIDEO_SAMPLE_FPS \
|
| 224 |
+
--data.video.video-size-pixels=$VIDEO_SIZE_PIXELS \
|
| 225 |
+
--data.video.video-decoder=$VIDEO_DECODER \
|
| 226 |
+
--data.video.video-cache-size=$VIDEO_CACHE_SIZE \
|
| 227 |
+
--data.video.sources-json=$PSI_VIDEO_SOURCES_JSON \
|
| 228 |
+
--data.video-clip-text-ratio=$PSI_VIDEO_CLIP_TEXT_TASK_WEIGHT \
|
| 229 |
+
--data.video-clip-only-ratio=$PSI_VIDEO_CLIP_ONLY_TASK_WEIGHT \
|
| 230 |
+
--data.video-clip.sources-json=$PSI_VIDEO_CLIP_SOURCES_JSON \
|
| 231 |
+
--data.video-clip.sample-stride=1 \
|
| 232 |
+
--data.video-clip.motion-fps=50 \
|
| 233 |
+
--data.video-clip.video-sample-fps=$VIDEO_CLIP_SAMPLE_FPS \
|
| 234 |
+
--data.video-clip.video-size-pixels=$VIDEO_SIZE_PIXELS \
|
| 235 |
+
--data.video-clip.video-decoder=$VIDEO_DECODER \
|
| 236 |
+
--data.video-clip.video-cache-size=$VIDEO_CACHE_SIZE \
|
| 237 |
+
--data.video-clip.action-exec-horizon=$ACTION_EXEC_HORIZON \
|
| 238 |
+
--data.sampler=batch_mixture \
|
| 239 |
+
--data.val-task=audio,audio_only \
|
| 240 |
+
--data.transform.repack.num-past-frames=$NUM_PAST_FRAMES \
|
| 241 |
+
--data.transform.repack.textop-actions \
|
| 242 |
+
--data.transform.repack.action-chunk-size=$ACTION_CHUNK_SIZE \
|
| 243 |
+
--data.transform.repack.pad-action-dim=39 \
|
| 244 |
+
--data.transform.repack.pad-state-dim=39 \
|
| 245 |
+
--data.transform.field.action_norm_type=bounds_q99 \
|
| 246 |
+
--data.transform.field.stat-path=assets/stats/seed_all_g1_motion39_textop_stats.json \
|
| 247 |
+
--data.transform.field.normalize-state \
|
| 248 |
+
--data.transform.field.pad-action-dim=39 \
|
| 249 |
+
--data.transform.field.pad-state-dim=39 \
|
| 250 |
+
--data.transform.model.resize.size 240 320 \
|
| 251 |
+
--data.transform.model.center_crop.size 240 320 \
|
| 252 |
+
--data.transform.model.no-img-aug \
|
| 253 |
+
--data.transform.model.text-only \
|
| 254 |
+
--data.transform.model.audio-only \
|
| 255 |
+
--data.transform.model.video-only \
|
| 256 |
+
--data.transform.model.video-size-pixels=$VIDEO_SIZE_PIXELS \
|
| 257 |
+
--model.model_name_or_path=$PSI_PRETRAIN_BACKBONE_PATH \
|
| 258 |
+
--model.noise-scheduler=flow \
|
| 259 |
+
--model.n_conditions=0 \
|
| 260 |
+
--model.action-chunk-size=$ACTION_CHUNK_SIZE \
|
| 261 |
+
--model.action-dim=39 \
|
| 262 |
+
--model.action-exec-horizon=$ACTION_EXEC_HORIZON \
|
| 263 |
+
--model.observation-horizon=$STATE_HORIZON \
|
| 264 |
+
--model.odim=39 \
|
| 265 |
+
--model.view_feature_dim=2048 \
|
| 266 |
+
--model.use-future-progress-embedding \
|
| 267 |
+
--model.future-progress-dim=$PROGRESS_DIM \
|
| 268 |
+
--model.tune-vlm \
|
| 269 |
+
--model.lang-backbone-lr=1e-5 \
|
| 270 |
+
--model.no-use_film \
|
| 271 |
+
--model.no-combined_temb \
|
| 272 |
+
--model.rtc \
|
| 273 |
+
--model.max-delay=11 \
|
| 274 |
+
--train.max-grad-value=1 \
|
| 275 |
+
--train.loss-group-weight.joint-abs=1.0 \
|
| 276 |
+
--train.loss-group-weight.root-delta-xy=3.0 \
|
| 277 |
+
--train.loss-group-weight.root-rot5-delta=10.0 \
|
| 278 |
+
--train.loss-group-weight.root-height=15.0 \
|
| 279 |
+
--train.loss-group-weight.foot-contact=0.1
|
| 280 |
+
"
|
| 281 |
+
torchrun --nproc_per_node=$NPROC_PER_NODE --master_port=29500 scripts/train.py \
|
| 282 |
+
$args "$@"
|
posttrain-v1-psi0-motion-distill.sh
ADDED
|
@@ -0,0 +1,349 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
pkill -f monitor.py
|
| 2 |
+
|
| 3 |
+
cd /cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/Psi0-yth-v18
|
| 4 |
+
export PYTHONPATH=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/Psi0-yth-v18/src
|
| 5 |
+
export CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7
|
| 6 |
+
export PSI_OUTPUT_DIR=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/Psi0-yth-v18/runs
|
| 7 |
+
export PSI_LOG_REPORT_TO=${PSI_LOG_REPORT_TO:-wandb}
|
| 8 |
+
|
| 9 |
+
source .venv-psi/bin/activate
|
| 10 |
+
[ -f .env ] && source .env
|
| 11 |
+
|
| 12 |
+
export WANDB_API_KEY="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"
|
| 13 |
+
|
| 14 |
+
export OMP_NUM_THREADS=8
|
| 15 |
+
export TF_CPP_MIN_LOG_LEVEL=3
|
| 16 |
+
|
| 17 |
+
compact_json() {
|
| 18 |
+
python -c 'import json, sys; print(json.dumps(json.load(sys.stdin), separators=(",", ":")))'
|
| 19 |
+
}
|
| 20 |
+
|
| 21 |
+
STAGE=${PSI_MOTION_DISTILL_STAGE:-teacher}
|
| 22 |
+
if [ "$1" = "teacher" ] || [ "$1" = "student" ] || [ "$1" = "cache" ]; then
|
| 23 |
+
STAGE=$1
|
| 24 |
+
shift
|
| 25 |
+
fi
|
| 26 |
+
|
| 27 |
+
PSI_SEED_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/Psi0/dataset/seed_all
|
| 28 |
+
PSI_SEED_METADATA=metadata/seed_metadata_v004.csv
|
| 29 |
+
|
| 30 |
+
PSI_DANCE_MOTION_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/GR00T-WholeBodyControl/gear_sonic_deploy/reference/converted/psi0_dance/gvhmr
|
| 31 |
+
PSI_DANCE_AUDIO_VIDEO_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/dance_1723_video_psi0/dance_1723_video_10s
|
| 32 |
+
PSI_DANCE_VIDEO_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/dance_1723_video_psi0/dance_1723_video_10s
|
| 33 |
+
PSI_DANCE_VIDEO_EMBED_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/dance_1723_video_psi0/dance_1723_video_10s_embeds
|
| 34 |
+
PSI_VIDEO_DEMO_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/videogen/motion_example_0506_demo
|
| 35 |
+
PSI_QWEN3_TTS_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/qwen3-tts
|
| 36 |
+
PSI_VIDEO_CLIP_DATASET_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/videogen/psi0_video_clip_50f_rand8_full
|
| 37 |
+
PSI_DANCE_VIDEO_CLIP_ROOT=$PSI_VIDEO_CLIP_DATASET_ROOT/dance_1723_video_10s
|
| 38 |
+
PSI_MOTION_EXAMPLE_VIDEO_CLIP_ROOT=$PSI_VIDEO_CLIP_DATASET_ROOT/motion_example_0506_demo
|
| 39 |
+
PSI_MOTION_EXAMPLE_VIDEO_EMBED_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/videogen/motion_example_0506_demo_video_embeds
|
| 40 |
+
|
| 41 |
+
if [ -z "$PSI_AUDIO_SOURCES_JSON" ]; then
|
| 42 |
+
PSI_AUDIO_SOURCES_JSON=$(compact_json <<JSON
|
| 43 |
+
[
|
| 44 |
+
{
|
| 45 |
+
"name": "psi0_dance_audio",
|
| 46 |
+
"root_dir": "$PSI_DANCE_MOTION_ROOT",
|
| 47 |
+
"video_dir": "$PSI_DANCE_AUDIO_VIDEO_ROOT",
|
| 48 |
+
"dataset_name": "seed-audio-g1"
|
| 49 |
+
}
|
| 50 |
+
]
|
| 51 |
+
JSON
|
| 52 |
+
)
|
| 53 |
+
fi
|
| 54 |
+
|
| 55 |
+
if [ -z "$PSI_AUDIO_COMMAND_SOURCES_JSON" ]; then
|
| 56 |
+
PSI_AUDIO_COMMAND_SOURCES_JSON=$(compact_json <<JSON
|
| 57 |
+
[
|
| 58 |
+
{
|
| 59 |
+
"name": "qwen3_tts_seed_audio_command",
|
| 60 |
+
"root_dir": "$PSI_SEED_ROOT",
|
| 61 |
+
"dataset_root": "$PSI_QWEN3_TTS_ROOT",
|
| 62 |
+
"metadata_jsonl": "metadata.jsonl",
|
| 63 |
+
"motion_dir_key": "filename",
|
| 64 |
+
"audio_path_key": "audio_path",
|
| 65 |
+
"instruction_key": "content_short_description",
|
| 66 |
+
"motion_metadata_csv": "$PSI_SEED_METADATA",
|
| 67 |
+
"motion_metadata_key": "filename",
|
| 68 |
+
"motion_metadata_path_key": "move_g1_path",
|
| 69 |
+
"audio_file_extensions": ".wav",
|
| 70 |
+
"audio_task": "command",
|
| 71 |
+
"dataset_name": "seed-audio-command-g1"
|
| 72 |
+
}
|
| 73 |
+
]
|
| 74 |
+
JSON
|
| 75 |
+
)
|
| 76 |
+
fi
|
| 77 |
+
|
| 78 |
+
if [ -z "$PSI_VIDEO_SOURCES_JSON" ]; then
|
| 79 |
+
PSI_VIDEO_SOURCES_JSON=$(compact_json <<JSON
|
| 80 |
+
[
|
| 81 |
+
{
|
| 82 |
+
"name": "psi0_dance_gvhmr",
|
| 83 |
+
"root_dir": "$PSI_DANCE_MOTION_ROOT",
|
| 84 |
+
"video_dir": "$PSI_DANCE_VIDEO_ROOT",
|
| 85 |
+
"video_embed_dir": "$PSI_DANCE_VIDEO_EMBED_ROOT",
|
| 86 |
+
"dataset_name": "seed-video-g1"
|
| 87 |
+
},
|
| 88 |
+
{
|
| 89 |
+
"name": "motion_example_0506_demo",
|
| 90 |
+
"dataset_root": "$PSI_VIDEO_DEMO_ROOT",
|
| 91 |
+
"metadata_jsonl": "metadata.jsonl",
|
| 92 |
+
"motion_dir_key": "converted_path",
|
| 93 |
+
"video_path_key": "video_path",
|
| 94 |
+
"instruction_key": "video_caption",
|
| 95 |
+
"video_embed_dir": "$PSI_MOTION_EXAMPLE_VIDEO_EMBED_ROOT",
|
| 96 |
+
"dataset_name": "motion-example-video-g1"
|
| 97 |
+
}
|
| 98 |
+
]
|
| 99 |
+
JSON
|
| 100 |
+
)
|
| 101 |
+
fi
|
| 102 |
+
|
| 103 |
+
if [ -z "$PSI_VIDEO_CLIP_SOURCES_JSON" ]; then
|
| 104 |
+
PSI_VIDEO_CLIP_SOURCES_JSON=$(compact_json <<JSON
|
| 105 |
+
[
|
| 106 |
+
{
|
| 107 |
+
"name": "dance_1723_video_10s_clip50",
|
| 108 |
+
"dataset_root": "$PSI_DANCE_VIDEO_CLIP_ROOT",
|
| 109 |
+
"metadata_jsonl": "metadata.jsonl",
|
| 110 |
+
"motion_dir_key": "converted_path",
|
| 111 |
+
"video_path_key": "video_path",
|
| 112 |
+
"instruction_key": "video_caption",
|
| 113 |
+
"video_embed_dir": "$PSI_DANCE_VIDEO_CLIP_ROOT/embeds",
|
| 114 |
+
"dataset_name": "dance-video-clip-g1"
|
| 115 |
+
},
|
| 116 |
+
{
|
| 117 |
+
"name": "motion_example_0506_demo_clip50",
|
| 118 |
+
"dataset_root": "$PSI_MOTION_EXAMPLE_VIDEO_CLIP_ROOT",
|
| 119 |
+
"metadata_jsonl": "metadata.jsonl",
|
| 120 |
+
"motion_dir_key": "converted_path",
|
| 121 |
+
"video_path_key": "video_path",
|
| 122 |
+
"instruction_key": "video_caption",
|
| 123 |
+
"video_embed_dir": "$PSI_MOTION_EXAMPLE_VIDEO_CLIP_ROOT/embeds",
|
| 124 |
+
"dataset_name": "motion-example-video-clip-g1"
|
| 125 |
+
}
|
| 126 |
+
]
|
| 127 |
+
JSON
|
| 128 |
+
)
|
| 129 |
+
fi
|
| 130 |
+
|
| 131 |
+
PSI_TEXT_TASK_WEIGHT=0
|
| 132 |
+
PSI_AUDIO_TEXT_TASK_WEIGHT=0
|
| 133 |
+
PSI_AUDIO_ONLY_TASK_WEIGHT=0
|
| 134 |
+
PSI_AUDIO_COMMAND_TEXT_TASK_WEIGHT=0
|
| 135 |
+
PSI_AUDIO_COMMAND_ONLY_TASK_WEIGHT=0
|
| 136 |
+
PSI_VIDEO_TEXT_TASK_WEIGHT=0
|
| 137 |
+
PSI_VIDEO_ONLY_TASK_WEIGHT=0
|
| 138 |
+
PSI_VIDEO_CLIP_TEXT_TASK_WEIGHT=0.5
|
| 139 |
+
PSI_VIDEO_CLIP_ONLY_TASK_WEIGHT=0.5
|
| 140 |
+
|
| 141 |
+
PSI_PRETRAIN_BACKBONE_PATH=/cpfs01/VideoGen/public_ckpt/Qwen/Qwen2.5-Omni-3B
|
| 142 |
+
NUM_PAST_FRAMES=0
|
| 143 |
+
STATE_HORIZON=$((NUM_PAST_FRAMES + 1))
|
| 144 |
+
ACTION_CHUNK_SIZE=30
|
| 145 |
+
ACTION_EXEC_HORIZON=20
|
| 146 |
+
MOTION_ENCODER_INPUT_HORIZON=${PSI_MOTION_ENCODER_INPUT_HORIZON:-$ACTION_CHUNK_SIZE}
|
| 147 |
+
PROGRESS_DIM=5
|
| 148 |
+
VIDEO_SAMPLE_FPS=1.0
|
| 149 |
+
VIDEO_CLIP_SAMPLE_FPS=8.0
|
| 150 |
+
VIDEO_SIZE_PIXELS=50176
|
| 151 |
+
VIDEO_DECODER=ffmpeg
|
| 152 |
+
VIDEO_CACHE_SIZE=64
|
| 153 |
+
DATALOADER_NUM_WORKERS=4
|
| 154 |
+
VAL_DATALOADER_NUM_WORKERS=2
|
| 155 |
+
MAX_TRAINING_STEPS=${PSI_MAX_TRAINING_STEPS:-1000000}
|
| 156 |
+
CHECKPOINTING_STEPS=${PSI_CHECKPOINTING_STEPS:-1000}
|
| 157 |
+
VALIDATION_STEPS=${PSI_VALIDATION_STEPS:-1000}
|
| 158 |
+
TEACHER_CHECKPOINT=${PSI_TEACHER_CHECKPOINT:-}
|
| 159 |
+
MOTION_LATENT_NUM_TOKENS=64
|
| 160 |
+
MOTION_LATENT_TOKEN=${PSI_MOTION_LATENT_TOKEN:-"<|motion_latent|>"}
|
| 161 |
+
MOTION_LATENT_TEACHER_ID=psi0_motion_distill
|
| 162 |
+
MOTION_LATENT_CACHE_DIR=${PSI_MOTION_LATENT_CACHE_DIR:-$PSI_VIDEO_CLIP_DATASET_ROOT/motion_latents/$MOTION_LATENT_TEACHER_ID}
|
| 163 |
+
STUDENT_VLM_SCOPE=${PSI_STUDENT_VLM_SCOPE:-video}
|
| 164 |
+
STUDENT_HIDDEN_LOSS_WEIGHT=${PSI_STUDENT_HIDDEN_LOSS_WEIGHT:-1.0}
|
| 165 |
+
STUDENT_HIDDEN_COSINE_LOSS_WEIGHT=${PSI_STUDENT_HIDDEN_COSINE_LOSS_WEIGHT:-0}
|
| 166 |
+
STUDENT_ACTION_LOSS_WEIGHT=${PSI_STUDENT_ACTION_LOSS_WEIGHT:-0}
|
| 167 |
+
TRAIN_BATCH_SIZE=64
|
| 168 |
+
VAL_BATCH_SIZE=64
|
| 169 |
+
if [ "$STAGE" = "student" ]; then
|
| 170 |
+
TRAIN_BATCH_SIZE=32
|
| 171 |
+
VAL_BATCH_SIZE=32
|
| 172 |
+
fi
|
| 173 |
+
|
| 174 |
+
NPROC_PER_NODE=$(echo $CUDA_VISIBLE_DEVICES | tr ',' '\n' | wc -l)
|
| 175 |
+
|
| 176 |
+
if [ "$STAGE" = "student" ] || [ "$STAGE" = "cache" ]; then
|
| 177 |
+
if [ -z "$TEACHER_CHECKPOINT" ]; then
|
| 178 |
+
echo "Set PSI_TEACHER_CHECKPOINT to the stage-1 run dir or checkpoint dir."
|
| 179 |
+
exit 1
|
| 180 |
+
fi
|
| 181 |
+
fi
|
| 182 |
+
|
| 183 |
+
echo "Motion distill stage: $STAGE"
|
| 184 |
+
echo "Training with $NPROC_PER_NODE GPU(s): $CUDA_VISIBLE_DEVICES"
|
| 185 |
+
echo "Video clip sources: $PSI_VIDEO_CLIP_SOURCES_JSON"
|
| 186 |
+
echo "Motion encoder input horizon: $MOTION_ENCODER_INPUT_HORIZON"
|
| 187 |
+
echo "Teacher checkpoint: $TEACHER_CHECKPOINT"
|
| 188 |
+
echo "Motion latent cache dir: $MOTION_LATENT_CACHE_DIR"
|
| 189 |
+
echo "Motion latent tokens: $MOTION_LATENT_NUM_TOKENS x $MOTION_LATENT_TOKEN"
|
| 190 |
+
echo "Student VLM scope: $STUDENT_VLM_SCOPE"
|
| 191 |
+
echo "Student hidden loss weight: $STUDENT_HIDDEN_LOSS_WEIGHT"
|
| 192 |
+
echo "Student hidden cosine loss weight: $STUDENT_HIDDEN_COSINE_LOSS_WEIGHT"
|
| 193 |
+
echo "Student action loss weight: $STUDENT_ACTION_LOSS_WEIGHT"
|
| 194 |
+
echo "Per-GPU batch size: train=$TRAIN_BATCH_SIZE val=$VAL_BATCH_SIZE"
|
| 195 |
+
echo "Backbone path: $PSI_PRETRAIN_BACKBONE_PATH"
|
| 196 |
+
echo "Output dir: $PSI_OUTPUT_DIR"
|
| 197 |
+
|
| 198 |
+
CONFIG_STAGE=$STAGE
|
| 199 |
+
if [ "$STAGE" = "cache" ]; then
|
| 200 |
+
CONFIG_STAGE=teacher
|
| 201 |
+
fi
|
| 202 |
+
|
| 203 |
+
stage_args="--model.motion-distill-stage=$CONFIG_STAGE \
|
| 204 |
+
--model.motion-encoder-input-horizon=$MOTION_ENCODER_INPUT_HORIZON \
|
| 205 |
+
--model.motion-latent-num-tokens=$MOTION_LATENT_NUM_TOKENS \
|
| 206 |
+
--model.motion-latent-token=$MOTION_LATENT_TOKEN \
|
| 207 |
+
--model.motion-encoder-hidden-dim=2048"
|
| 208 |
+
|
| 209 |
+
if [ "$STAGE" = "teacher" ]; then
|
| 210 |
+
stage_args="$stage_args --model.no-tune-vlm"
|
| 211 |
+
elif [ "$STAGE" = "student" ]; then
|
| 212 |
+
stage_args="$stage_args \
|
| 213 |
+
--model.tune-vlm \
|
| 214 |
+
--model.motion-teacher-checkpoint=$TEACHER_CHECKPOINT \
|
| 215 |
+
--model.motion-latent-cache-dir=$MOTION_LATENT_CACHE_DIR \
|
| 216 |
+
--model.motion-distill-vlm-train-scope=$STUDENT_VLM_SCOPE \
|
| 217 |
+
--model.motion-hidden-loss-weight=$STUDENT_HIDDEN_LOSS_WEIGHT \
|
| 218 |
+
--model.motion-hidden-cosine-loss-weight=$STUDENT_HIDDEN_COSINE_LOSS_WEIGHT \
|
| 219 |
+
--model.motion-student-action-loss-weight=$STUDENT_ACTION_LOSS_WEIGHT \
|
| 220 |
+
--model.motion-latent-cache-strict"
|
| 221 |
+
elif [ "$STAGE" = "cache" ]; then
|
| 222 |
+
stage_args="$stage_args \
|
| 223 |
+
--model.no-tune-vlm \
|
| 224 |
+
--model.motion-teacher-checkpoint=$TEACHER_CHECKPOINT \
|
| 225 |
+
--model.motion-latent-cache-dir=$MOTION_LATENT_CACHE_DIR"
|
| 226 |
+
else
|
| 227 |
+
echo "Unknown stage: $STAGE"
|
| 228 |
+
exit 1
|
| 229 |
+
fi
|
| 230 |
+
|
| 231 |
+
args="posttrain_v1_seed_text_audio_video_mix_psi0_motion_distill_config \
|
| 232 |
+
--seed=292285 \
|
| 233 |
+
--exp=posttrain \
|
| 234 |
+
--timestamp=$(date +"%y%m%d%H%M") \
|
| 235 |
+
--train.name=posttrain_v1_motion_distill \
|
| 236 |
+
--train.data_parallel=ddp \
|
| 237 |
+
--train.mixed_precision=bf16 \
|
| 238 |
+
--train.train_batch_size=$TRAIN_BATCH_SIZE \
|
| 239 |
+
--train.val_batch_size=$VAL_BATCH_SIZE \
|
| 240 |
+
--train.num-workers=$DATALOADER_NUM_WORKERS \
|
| 241 |
+
--train.val-num-workers=$VAL_DATALOADER_NUM_WORKERS \
|
| 242 |
+
--train.resume_from_checkpoint=None \
|
| 243 |
+
--train.max_checkpoints_to_keep=1 \
|
| 244 |
+
--train.gradient_accumulation_steps=1 \
|
| 245 |
+
--train.learning_rate=1e-4 \
|
| 246 |
+
--train.max_training_steps=$MAX_TRAINING_STEPS \
|
| 247 |
+
--train.warmup_ratio=None \
|
| 248 |
+
--train.warmup_steps=1000 \
|
| 249 |
+
--train.checkpointing_steps=$CHECKPOINTING_STEPS \
|
| 250 |
+
--train.validation_steps=$VALIDATION_STEPS \
|
| 251 |
+
--train.val_num_batches=20 \
|
| 252 |
+
--train.max_grad_norm=1.0 \
|
| 253 |
+
--train.lr_scheduler_type=cosine_with_min_lr \
|
| 254 |
+
--train.lr_scheduler_kwargs.betas 0.9 0.999 \
|
| 255 |
+
--train.lr_scheduler_kwargs.weight_decay=0 \
|
| 256 |
+
--train.lr_scheduler_kwargs.eps=1e-8 \
|
| 257 |
+
--log.report_to=$PSI_LOG_REPORT_TO \
|
| 258 |
+
--train.output_dir=$PSI_OUTPUT_DIR \
|
| 259 |
+
--data.text.root-dir=$PSI_SEED_ROOT \
|
| 260 |
+
--data.text.metadata-csv=$PSI_SEED_METADATA \
|
| 261 |
+
--data.text-ratio=$PSI_TEXT_TASK_WEIGHT \
|
| 262 |
+
--data.audio.sources-json=$PSI_AUDIO_SOURCES_JSON \
|
| 263 |
+
--data.audio-ratio=$PSI_AUDIO_TEXT_TASK_WEIGHT \
|
| 264 |
+
--data.audio-only-ratio=$PSI_AUDIO_ONLY_TASK_WEIGHT \
|
| 265 |
+
--data.audio.sample-stride=1 \
|
| 266 |
+
--data.audio.motion-fps=50 \
|
| 267 |
+
--data.audio.audio-sampling-rate=16000 \
|
| 268 |
+
--data.audio.audio-window-seconds=1.0 \
|
| 269 |
+
--data.audio.audio-left-context-seconds=0.0 \
|
| 270 |
+
--data.audio.full-audio \
|
| 271 |
+
--data.audio-command.sources-json=$PSI_AUDIO_COMMAND_SOURCES_JSON \
|
| 272 |
+
--data.audio-command-ratio=$PSI_AUDIO_COMMAND_TEXT_TASK_WEIGHT \
|
| 273 |
+
--data.audio-command-only-ratio=$PSI_AUDIO_COMMAND_ONLY_TASK_WEIGHT \
|
| 274 |
+
--data.audio-command.sample-stride=1 \
|
| 275 |
+
--data.audio-command.motion-fps=50 \
|
| 276 |
+
--data.audio-command.audio-sampling-rate=16000 \
|
| 277 |
+
--data.audio-command.audio-window-seconds=1.0 \
|
| 278 |
+
--data.audio-command.audio-left-context-seconds=0.0 \
|
| 279 |
+
--data.audio-command.full-audio \
|
| 280 |
+
--data.video-ratio=$PSI_VIDEO_TEXT_TASK_WEIGHT \
|
| 281 |
+
--data.video-only-ratio=$PSI_VIDEO_ONLY_TASK_WEIGHT \
|
| 282 |
+
--data.video.sample-stride=1 \
|
| 283 |
+
--data.video.motion-fps=50 \
|
| 284 |
+
--data.video.video-sample-fps=$VIDEO_SAMPLE_FPS \
|
| 285 |
+
--data.video.video-size-pixels=$VIDEO_SIZE_PIXELS \
|
| 286 |
+
--data.video.video-decoder=$VIDEO_DECODER \
|
| 287 |
+
--data.video.video-cache-size=$VIDEO_CACHE_SIZE \
|
| 288 |
+
--data.video.sources-json=$PSI_VIDEO_SOURCES_JSON \
|
| 289 |
+
--data.video-clip-text-ratio=$PSI_VIDEO_CLIP_TEXT_TASK_WEIGHT \
|
| 290 |
+
--data.video-clip-only-ratio=$PSI_VIDEO_CLIP_ONLY_TASK_WEIGHT \
|
| 291 |
+
--data.video-clip.sources-json=$PSI_VIDEO_CLIP_SOURCES_JSON \
|
| 292 |
+
--data.video-clip.sample-stride=1 \
|
| 293 |
+
--data.video-clip.motion-fps=50 \
|
| 294 |
+
--data.video-clip.video-sample-fps=$VIDEO_CLIP_SAMPLE_FPS \
|
| 295 |
+
--data.video-clip.video-size-pixels=$VIDEO_SIZE_PIXELS \
|
| 296 |
+
--data.video-clip.video-decoder=$VIDEO_DECODER \
|
| 297 |
+
--data.video-clip.video-cache-size=$VIDEO_CACHE_SIZE \
|
| 298 |
+
--data.video-clip.action-exec-horizon=$ACTION_EXEC_HORIZON \
|
| 299 |
+
--data.sampler=batch_mixture \
|
| 300 |
+
--data.val-task=video_clip,video_clip_only \
|
| 301 |
+
--data.transform.repack.num-past-frames=$NUM_PAST_FRAMES \
|
| 302 |
+
--data.transform.repack.textop-actions \
|
| 303 |
+
--data.transform.repack.action-chunk-size=$ACTION_CHUNK_SIZE \
|
| 304 |
+
--data.transform.repack.pad-action-dim=39 \
|
| 305 |
+
--data.transform.repack.pad-state-dim=39 \
|
| 306 |
+
--data.transform.field.action_norm_type=bounds_q99 \
|
| 307 |
+
--data.transform.field.stat-path=assets/stats/seed_all_g1_motion39_textop_stats.json \
|
| 308 |
+
--data.transform.field.normalize-state \
|
| 309 |
+
--data.transform.field.pad-action-dim=39 \
|
| 310 |
+
--data.transform.field.pad-state-dim=39 \
|
| 311 |
+
--data.transform.model.resize.size 240 320 \
|
| 312 |
+
--data.transform.model.center_crop.size 240 320 \
|
| 313 |
+
--data.transform.model.no-img-aug \
|
| 314 |
+
--data.transform.model.text-only \
|
| 315 |
+
--data.transform.model.audio-only \
|
| 316 |
+
--data.transform.model.video-only \
|
| 317 |
+
--data.transform.model.video-size-pixels=$VIDEO_SIZE_PIXELS \
|
| 318 |
+
--data.transform.model.motion-latent-token=$MOTION_LATENT_TOKEN \
|
| 319 |
+
--data.transform.model.motion-latent-num-tokens=$MOTION_LATENT_NUM_TOKENS \
|
| 320 |
+
--model.model_name_or_path=$PSI_PRETRAIN_BACKBONE_PATH \
|
| 321 |
+
--model.noise-scheduler=flow \
|
| 322 |
+
--model.n_conditions=0 \
|
| 323 |
+
--model.action-chunk-size=$ACTION_CHUNK_SIZE \
|
| 324 |
+
--model.action-dim=39 \
|
| 325 |
+
--model.action-exec-horizon=$ACTION_EXEC_HORIZON \
|
| 326 |
+
--model.observation-horizon=$STATE_HORIZON \
|
| 327 |
+
--model.odim=39 \
|
| 328 |
+
--model.view_feature_dim=2048 \
|
| 329 |
+
--model.use-future-progress-embedding \
|
| 330 |
+
--model.future-progress-dim=$PROGRESS_DIM \
|
| 331 |
+
--model.no-use_film \
|
| 332 |
+
--model.no-combined_temb \
|
| 333 |
+
--model.rtc \
|
| 334 |
+
--model.max-delay=11 \
|
| 335 |
+
--train.max-grad-value=1 \
|
| 336 |
+
--train.loss-group-weight.joint-abs=1.0 \
|
| 337 |
+
--train.loss-group-weight.root-delta-xy=1.0 \
|
| 338 |
+
--train.loss-group-weight.root-rot5-delta=1.0 \
|
| 339 |
+
--train.loss-group-weight.root-height=1.0 \
|
| 340 |
+
--train.loss-group-weight.foot-contact=1.0 \
|
| 341 |
+
$stage_args
|
| 342 |
+
"
|
| 343 |
+
|
| 344 |
+
if [ "$STAGE" = "cache" ]; then
|
| 345 |
+
python scripts/save_motion_teacher_hidden_cache.py $args "$@"
|
| 346 |
+
else
|
| 347 |
+
torchrun --nproc_per_node=$NPROC_PER_NODE --master_port=${MASTER_PORT:-29510} scripts/train.py \
|
| 348 |
+
$args "$@"
|
| 349 |
+
fi
|
posttrain-v1-psi0-text-ckpt-continue-video-clip-tune-vlm-direct-dense.sh
ADDED
|
@@ -0,0 +1,305 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
pkill -f monitor.py
|
| 2 |
+
|
| 3 |
+
cd /cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/Psi0-yth-v18
|
| 4 |
+
export PYTHONPATH=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/Psi0-yth-v18/src
|
| 5 |
+
export CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7
|
| 6 |
+
export PSI_OUTPUT_DIR=${PSI_OUTPUT_DIR:-/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/Psi0-yth-v18/runs}
|
| 7 |
+
export PSI_LOG_REPORT_TO=wandb
|
| 8 |
+
|
| 9 |
+
source /cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/Psi0-yth-v15/.venv-psi/bin/activate
|
| 10 |
+
[ -f .env ] && source .env
|
| 11 |
+
|
| 12 |
+
export WANDB_API_KEY="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"
|
| 13 |
+
|
| 14 |
+
export OMP_NUM_THREADS=8
|
| 15 |
+
export TF_CPP_MIN_LOG_LEVEL=3
|
| 16 |
+
|
| 17 |
+
# Keep JSON sources readable in this script, but pass compact JSON to tyro.
|
| 18 |
+
compact_json() {
|
| 19 |
+
python -c 'import json, sys; print(json.dumps(json.load(sys.stdin), separators=(",", ":")))'
|
| 20 |
+
}
|
| 21 |
+
|
| 22 |
+
PSI_SEED_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/Psi0/dataset/seed_all
|
| 23 |
+
PSI_SEED_METADATA=metadata/seed_metadata_v004.csv
|
| 24 |
+
|
| 25 |
+
PSI_DANCE_MOTION_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/GR00T-WholeBodyControl/gear_sonic_deploy/reference/converted/psi0_dance/gvhmr
|
| 26 |
+
PSI_DANCE_AUDIO_VIDEO_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/dance_1723_video_psi0/dance_1723_video_10s
|
| 27 |
+
PSI_DANCE_VIDEO_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/dance_1723_video_psi0/dance_1723_video_10s
|
| 28 |
+
PSI_DANCE_VIDEO_EMBED_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/dance_1723_video_psi0/dance_1723_video_10s_embeds
|
| 29 |
+
PSI_VIDEO_DEMO_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/videogen/motion_example_0506_demo
|
| 30 |
+
PSI_QWEN3_TTS_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/qwen3-tts
|
| 31 |
+
PSI_VIDEO_CLIP_DATASET_ROOT=${PSI_VIDEO_CLIP_DATASET_ROOT:-/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/videogen/psi0_video_clip_50f_dense_stride1_direct_full}
|
| 32 |
+
PSI_DANCE_VIDEO_CLIP_ROOT=$PSI_VIDEO_CLIP_DATASET_ROOT/dance_1723_video_10s
|
| 33 |
+
PSI_MOTION_EXAMPLE_VIDEO_CLIP_ROOT=$PSI_VIDEO_CLIP_DATASET_ROOT/motion_example_0506_demo
|
| 34 |
+
PSI_DANCE_VIDEO_CLIP_EMBED_ROOT=${PSI_DANCE_VIDEO_CLIP_EMBED_ROOT:-$PSI_DANCE_VIDEO_CLIP_ROOT/embeds}
|
| 35 |
+
PSI_MOTION_EXAMPLE_VIDEO_CLIP_EMBED_ROOT=${PSI_MOTION_EXAMPLE_VIDEO_CLIP_EMBED_ROOT:-$PSI_MOTION_EXAMPLE_VIDEO_CLIP_ROOT/embeds}
|
| 36 |
+
PSI_MOTION_EXAMPLE_VIDEO_EMBED_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/videogen/motion_example_0506_demo_video_embeds
|
| 37 |
+
|
| 38 |
+
if [ -z "$PSI_AUDIO_SOURCES_JSON" ]; then
|
| 39 |
+
PSI_AUDIO_SOURCES_JSON=$(compact_json <<JSON
|
| 40 |
+
[
|
| 41 |
+
{
|
| 42 |
+
"name": "psi0_dance_audio",
|
| 43 |
+
"root_dir": "$PSI_DANCE_MOTION_ROOT",
|
| 44 |
+
"video_dir": "$PSI_DANCE_AUDIO_VIDEO_ROOT",
|
| 45 |
+
"dataset_name": "seed-audio-g1"
|
| 46 |
+
}
|
| 47 |
+
]
|
| 48 |
+
JSON
|
| 49 |
+
)
|
| 50 |
+
fi
|
| 51 |
+
|
| 52 |
+
if [ -z "$PSI_AUDIO_COMMAND_SOURCES_JSON" ]; then
|
| 53 |
+
PSI_AUDIO_COMMAND_SOURCES_JSON=$(compact_json <<JSON
|
| 54 |
+
[
|
| 55 |
+
{
|
| 56 |
+
"name": "qwen3_tts_seed_audio_command",
|
| 57 |
+
"root_dir": "$PSI_SEED_ROOT",
|
| 58 |
+
"dataset_root": "$PSI_QWEN3_TTS_ROOT",
|
| 59 |
+
"metadata_jsonl": "metadata.jsonl",
|
| 60 |
+
"motion_dir_key": "filename",
|
| 61 |
+
"audio_path_key": "audio_path",
|
| 62 |
+
"instruction_key": "content_short_description",
|
| 63 |
+
"motion_metadata_csv": "$PSI_SEED_METADATA",
|
| 64 |
+
"motion_metadata_key": "filename",
|
| 65 |
+
"motion_metadata_path_key": "move_g1_path",
|
| 66 |
+
"audio_file_extensions": ".wav",
|
| 67 |
+
"audio_task": "command",
|
| 68 |
+
"dataset_name": "seed-audio-command-g1"
|
| 69 |
+
}
|
| 70 |
+
]
|
| 71 |
+
JSON
|
| 72 |
+
)
|
| 73 |
+
fi
|
| 74 |
+
|
| 75 |
+
if [ -z "$PSI_VIDEO_SOURCES_JSON" ]; then
|
| 76 |
+
PSI_VIDEO_SOURCES_JSON=$(compact_json <<JSON
|
| 77 |
+
[
|
| 78 |
+
{
|
| 79 |
+
"name": "psi0_dance_gvhmr",
|
| 80 |
+
"root_dir": "$PSI_DANCE_MOTION_ROOT",
|
| 81 |
+
"video_dir": "$PSI_DANCE_VIDEO_ROOT",
|
| 82 |
+
"video_embed_dir": "$PSI_DANCE_VIDEO_EMBED_ROOT",
|
| 83 |
+
"dataset_name": "seed-video-g1"
|
| 84 |
+
},
|
| 85 |
+
{
|
| 86 |
+
"name": "motion_example_0506_demo",
|
| 87 |
+
"dataset_root": "$PSI_VIDEO_DEMO_ROOT",
|
| 88 |
+
"metadata_jsonl": "metadata.jsonl",
|
| 89 |
+
"motion_dir_key": "converted_path",
|
| 90 |
+
"video_path_key": "video_path",
|
| 91 |
+
"instruction_key": "video_caption",
|
| 92 |
+
"video_embed_dir": "$PSI_MOTION_EXAMPLE_VIDEO_EMBED_ROOT",
|
| 93 |
+
"dataset_name": "motion-example-video-g1"
|
| 94 |
+
}
|
| 95 |
+
]
|
| 96 |
+
JSON
|
| 97 |
+
)
|
| 98 |
+
fi
|
| 99 |
+
|
| 100 |
+
if [ -z "$PSI_VIDEO_CLIP_SOURCES_JSON" ]; then
|
| 101 |
+
PSI_VIDEO_CLIP_SOURCES_JSON=$(compact_json <<JSON
|
| 102 |
+
[
|
| 103 |
+
{
|
| 104 |
+
"name": "dance_1723_video_10s_clip50",
|
| 105 |
+
"dataset_root": "$PSI_DANCE_VIDEO_CLIP_ROOT",
|
| 106 |
+
"metadata_jsonl": "metadata.jsonl",
|
| 107 |
+
"motion_dir_key": "converted_path",
|
| 108 |
+
"video_path_key": "video_path",
|
| 109 |
+
"instruction_key": "video_caption",
|
| 110 |
+
"video_embed_dir": "$PSI_DANCE_VIDEO_CLIP_EMBED_ROOT",
|
| 111 |
+
"dataset_name": "dance-video-clip-g1"
|
| 112 |
+
},
|
| 113 |
+
{
|
| 114 |
+
"name": "motion_example_0506_demo_clip50",
|
| 115 |
+
"dataset_root": "$PSI_MOTION_EXAMPLE_VIDEO_CLIP_ROOT",
|
| 116 |
+
"metadata_jsonl": "metadata.jsonl",
|
| 117 |
+
"motion_dir_key": "converted_path",
|
| 118 |
+
"video_path_key": "video_path",
|
| 119 |
+
"instruction_key": "video_caption",
|
| 120 |
+
"video_embed_dir": "$PSI_MOTION_EXAMPLE_VIDEO_CLIP_EMBED_ROOT",
|
| 121 |
+
"dataset_name": "motion-example-video-clip-g1"
|
| 122 |
+
}
|
| 123 |
+
]
|
| 124 |
+
JSON
|
| 125 |
+
)
|
| 126 |
+
fi
|
| 127 |
+
|
| 128 |
+
PSI_RESUME_TEXT_CHECKPOINT=${PSI_RESUME_TEXT_CHECKPOINT:-/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/Psi0-yth-v15/runs/posttrain_v1_trimodal/posttrain.sg.flow1000.cwml.lr1.0e-04.b256.gpus8.2605310619/checkpoints/ckpt_280000}
|
| 129 |
+
|
| 130 |
+
PSI_TEXT_TASK_WEIGHT=0.5
|
| 131 |
+
PSI_AUDIO_TEXT_TASK_WEIGHT=0
|
| 132 |
+
PSI_AUDIO_ONLY_TASK_WEIGHT=0
|
| 133 |
+
PSI_AUDIO_COMMAND_TEXT_TASK_WEIGHT=0
|
| 134 |
+
PSI_AUDIO_COMMAND_ONLY_TASK_WEIGHT=0
|
| 135 |
+
PSI_VIDEO_TEXT_TASK_WEIGHT=0
|
| 136 |
+
PSI_VIDEO_ONLY_TASK_WEIGHT=0
|
| 137 |
+
PSI_VIDEO_CLIP_TEXT_TASK_WEIGHT=0.25
|
| 138 |
+
PSI_VIDEO_CLIP_ONLY_TASK_WEIGHT=0.25
|
| 139 |
+
|
| 140 |
+
PSI_PRETRAIN_BACKBONE_PATH=/cpfs01/VideoGen/public_ckpt/Qwen/Qwen2.5-Omni-3B
|
| 141 |
+
NUM_PAST_FRAMES=0
|
| 142 |
+
STATE_HORIZON=$((NUM_PAST_FRAMES + 1))
|
| 143 |
+
ACTION_CHUNK_SIZE=30
|
| 144 |
+
ACTION_EXEC_HORIZON=20
|
| 145 |
+
PROGRESS_DIM=5
|
| 146 |
+
VIDEO_SAMPLE_FPS=1.0
|
| 147 |
+
VIDEO_CLIP_SAMPLE_FPS=8.0
|
| 148 |
+
VIDEO_SIZE_PIXELS=50176
|
| 149 |
+
VIDEO_DECODER=ffmpeg
|
| 150 |
+
VIDEO_CACHE_SIZE=64
|
| 151 |
+
VIDEO_CLIP_MOTION_CACHE_SIZE=256
|
| 152 |
+
VIDEO_CLIP_TRUST_METADATA=${VIDEO_CLIP_TRUST_METADATA:-1}
|
| 153 |
+
VIDEO_CLIP_RECORDS_CACHE_DIR=${VIDEO_CLIP_RECORDS_CACHE_DIR:-$PSI_VIDEO_CLIP_DATASET_ROOT/.index_cache}
|
| 154 |
+
if [ "$VIDEO_CLIP_TRUST_METADATA" = "1" ] || [ "$VIDEO_CLIP_TRUST_METADATA" = "true" ] || [ "$VIDEO_CLIP_TRUST_METADATA" = "True" ]; then
|
| 155 |
+
VIDEO_CLIP_TRUST_METADATA_FLAG="--data.video-clip.trust-metadata"
|
| 156 |
+
else
|
| 157 |
+
VIDEO_CLIP_TRUST_METADATA_FLAG="--data.video-clip.no-trust-metadata"
|
| 158 |
+
fi
|
| 159 |
+
DATALOADER_NUM_WORKERS=4
|
| 160 |
+
VAL_DATALOADER_NUM_WORKERS=2
|
| 161 |
+
|
| 162 |
+
if [ ! -f "$PSI_RESUME_TEXT_CHECKPOINT/model.safetensors" ] || [ ! -f "$PSI_RESUME_TEXT_CHECKPOINT/optimizer.bin" ] || [ ! -f "$PSI_RESUME_TEXT_CHECKPOINT/scheduler.bin" ]; then
|
| 163 |
+
echo "ERROR: resume checkpoint is not a full training-state checkpoint: $PSI_RESUME_TEXT_CHECKPOINT" >&2
|
| 164 |
+
exit 1
|
| 165 |
+
fi
|
| 166 |
+
|
| 167 |
+
NPROC_PER_NODE=$(echo $CUDA_VISIBLE_DEVICES | tr ',' '\n' | wc -l)
|
| 168 |
+
|
| 169 |
+
echo "Training with $NPROC_PER_NODE GPU(s): $CUDA_VISIBLE_DEVICES"
|
| 170 |
+
echo "Text root: $PSI_SEED_ROOT"
|
| 171 |
+
echo "Dance motion root: $PSI_DANCE_MOTION_ROOT"
|
| 172 |
+
echo "Dance audio/video root: $PSI_DANCE_AUDIO_VIDEO_ROOT"
|
| 173 |
+
echo "Dance video root: $PSI_DANCE_VIDEO_ROOT"
|
| 174 |
+
echo "Dance video embed root: $PSI_DANCE_VIDEO_EMBED_ROOT"
|
| 175 |
+
echo "Qwen3-TTS acdudio command root: $PSI_QWEN3_TTS_ROOT"
|
| 176 |
+
echo "Audio sources: $PSI_AUDIO_SOURCES_JSON"
|
| 177 |
+
echo "Audio command sources: $PSI_AUDIO_COMMAND_SOURCES_JSON"
|
| 178 |
+
echo "Video sources: $PSI_VIDEO_SOURCES_JSON"
|
| 179 |
+
echo "Video clip sources: $PSI_VIDEO_CLIP_SOURCES_JSON"
|
| 180 |
+
echo "Task weights: text=$PSI_TEXT_TASK_WEIGHT audio_text=$PSI_AUDIO_TEXT_TASK_WEIGHT audio_only=$PSI_AUDIO_ONLY_TASK_WEIGHT audio_command_text=$PSI_AUDIO_COMMAND_TEXT_TASK_WEIGHT audio_command_only=$PSI_AUDIO_COMMAND_ONLY_TASK_WEIGHT video_text=$PSI_VIDEO_TEXT_TASK_WEIGHT video_only=$PSI_VIDEO_ONLY_TASK_WEIGHT video_clip_text=$PSI_VIDEO_CLIP_TEXT_TASK_WEIGHT video_clip_only=$PSI_VIDEO_CLIP_ONLY_TASK_WEIGHT"
|
| 181 |
+
echo "Resume full training state from: $PSI_RESUME_TEXT_CHECKPOINT"
|
| 182 |
+
echo "TextOp 39D action/state enabled"
|
| 183 |
+
echo "Future progress embedding: enabled, dim $PROGRESS_DIM"
|
| 184 |
+
echo "Backbone path: $PSI_PRETRAIN_BACKBONE_PATH"
|
| 185 |
+
echo "Output dir: $PSI_OUTPUT_DIR"
|
| 186 |
+
echo "Video clip trust metadata: $VIDEO_CLIP_TRUST_METADATA"
|
| 187 |
+
echo "Video clip records cache dir: $VIDEO_CLIP_RECORDS_CACHE_DIR"
|
| 188 |
+
|
| 189 |
+
args="posttrain_v1_seed_text_audio_video_mix_psi0_config \
|
| 190 |
+
--seed=292285 \
|
| 191 |
+
--exp=posttrain \
|
| 192 |
+
--timestamp=$(date +"%y%m%d%H%M") \
|
| 193 |
+
--train.name=posttrain_v1_trimodal \
|
| 194 |
+
--train.data_parallel=ddp \
|
| 195 |
+
--train.mixed_precision=bf16 \
|
| 196 |
+
--train.train_batch_size=32 \
|
| 197 |
+
--train.val_batch_size=64 \
|
| 198 |
+
--train.num-workers=$DATALOADER_NUM_WORKERS \
|
| 199 |
+
--train.val-num-workers=$VAL_DATALOADER_NUM_WORKERS \
|
| 200 |
+
--train.resume_from_checkpoint=$PSI_RESUME_TEXT_CHECKPOINT \
|
| 201 |
+
--train.resume-load-full-state \
|
| 202 |
+
--train.max_checkpoints_to_keep=1 \
|
| 203 |
+
--train.gradient_accumulation_steps=1 \
|
| 204 |
+
--train.learning_rate=1e-4 \
|
| 205 |
+
--train.max_training_steps=800000 \
|
| 206 |
+
--train.warmup_ratio=None \
|
| 207 |
+
--train.warmup_steps=1000 \
|
| 208 |
+
--train.checkpointing_steps=1000 \
|
| 209 |
+
--train.validation_steps=1000 \
|
| 210 |
+
--train.val_num_batches=20 \
|
| 211 |
+
--train.max_grad_norm=1.0 \
|
| 212 |
+
--train.lr_scheduler_type=cosine_with_min_lr \
|
| 213 |
+
--train.lr_scheduler_kwargs.betas 0.9 0.999 \
|
| 214 |
+
--train.lr_scheduler_kwargs.weight_decay=0 \
|
| 215 |
+
--train.lr_scheduler_kwargs.eps=1e-8 \
|
| 216 |
+
--log.report_to=$PSI_LOG_REPORT_TO \
|
| 217 |
+
--train.output_dir=$PSI_OUTPUT_DIR \
|
| 218 |
+
--data.text.root-dir=$PSI_SEED_ROOT \
|
| 219 |
+
--data.text.metadata-csv=$PSI_SEED_METADATA \
|
| 220 |
+
--data.text-ratio=$PSI_TEXT_TASK_WEIGHT \
|
| 221 |
+
--data.audio.sources-json=$PSI_AUDIO_SOURCES_JSON \
|
| 222 |
+
--data.audio-ratio=$PSI_AUDIO_TEXT_TASK_WEIGHT \
|
| 223 |
+
--data.audio-only-ratio=$PSI_AUDIO_ONLY_TASK_WEIGHT \
|
| 224 |
+
--data.audio.sample-stride=1 \
|
| 225 |
+
--data.audio.motion-fps=50 \
|
| 226 |
+
--data.audio.audio-sampling-rate=16000 \
|
| 227 |
+
--data.audio.audio-window-seconds=1.0 \
|
| 228 |
+
--data.audio.audio-left-context-seconds=0.0 \
|
| 229 |
+
--data.audio.full-audio \
|
| 230 |
+
--data.audio-command.sources-json=$PSI_AUDIO_COMMAND_SOURCES_JSON \
|
| 231 |
+
--data.audio-command-ratio=$PSI_AUDIO_COMMAND_TEXT_TASK_WEIGHT \
|
| 232 |
+
--data.audio-command-only-ratio=$PSI_AUDIO_COMMAND_ONLY_TASK_WEIGHT \
|
| 233 |
+
--data.audio-command.sample-stride=1 \
|
| 234 |
+
--data.audio-command.motion-fps=50 \
|
| 235 |
+
--data.audio-command.audio-sampling-rate=16000 \
|
| 236 |
+
--data.audio-command.audio-window-seconds=1.0 \
|
| 237 |
+
--data.audio-command.audio-left-context-seconds=0.0 \
|
| 238 |
+
--data.audio-command.full-audio \
|
| 239 |
+
--data.video-ratio=$PSI_VIDEO_TEXT_TASK_WEIGHT \
|
| 240 |
+
--data.video-only-ratio=$PSI_VIDEO_ONLY_TASK_WEIGHT \
|
| 241 |
+
--data.video.sample-stride=1 \
|
| 242 |
+
--data.video.motion-fps=50 \
|
| 243 |
+
--data.video.video-sample-fps=$VIDEO_SAMPLE_FPS \
|
| 244 |
+
--data.video.video-size-pixels=$VIDEO_SIZE_PIXELS \
|
| 245 |
+
--data.video.video-decoder=$VIDEO_DECODER \
|
| 246 |
+
--data.video.video-cache-size=$VIDEO_CACHE_SIZE \
|
| 247 |
+
--data.video.sources-json=$PSI_VIDEO_SOURCES_JSON \
|
| 248 |
+
--data.video-clip-text-ratio=$PSI_VIDEO_CLIP_TEXT_TASK_WEIGHT \
|
| 249 |
+
--data.video-clip-only-ratio=$PSI_VIDEO_CLIP_ONLY_TASK_WEIGHT \
|
| 250 |
+
--data.video-clip.sources-json=$PSI_VIDEO_CLIP_SOURCES_JSON \
|
| 251 |
+
--data.video-clip.sample-stride=1 \
|
| 252 |
+
--data.video-clip.motion-fps=50 \
|
| 253 |
+
--data.video-clip.video-sample-fps=$VIDEO_CLIP_SAMPLE_FPS \
|
| 254 |
+
--data.video-clip.video-size-pixels=$VIDEO_SIZE_PIXELS \
|
| 255 |
+
--data.video-clip.video-decoder=$VIDEO_DECODER \
|
| 256 |
+
--data.video-clip.video-cache-size=$VIDEO_CACHE_SIZE \
|
| 257 |
+
--data.video-clip.motion-cache-size=$VIDEO_CLIP_MOTION_CACHE_SIZE \
|
| 258 |
+
$VIDEO_CLIP_TRUST_METADATA_FLAG \
|
| 259 |
+
--data.video-clip.records-cache-dir=$VIDEO_CLIP_RECORDS_CACHE_DIR \
|
| 260 |
+
--data.video-clip.action-exec-horizon=$ACTION_EXEC_HORIZON \
|
| 261 |
+
--data.sampler=batch_mixture \
|
| 262 |
+
--data.val-task=text,video_clip,video_clip_only \
|
| 263 |
+
--data.transform.repack.num-past-frames=$NUM_PAST_FRAMES \
|
| 264 |
+
--data.transform.repack.textop-actions \
|
| 265 |
+
--data.transform.repack.action-chunk-size=$ACTION_CHUNK_SIZE \
|
| 266 |
+
--data.transform.repack.pad-action-dim=39 \
|
| 267 |
+
--data.transform.repack.pad-state-dim=39 \
|
| 268 |
+
--data.transform.field.action_norm_type=bounds_q99 \
|
| 269 |
+
--data.transform.field.stat-path=assets/stats/seed_all_g1_motion39_textop_stats.json \
|
| 270 |
+
--data.transform.field.normalize-state \
|
| 271 |
+
--data.transform.field.pad-action-dim=39 \
|
| 272 |
+
--data.transform.field.pad-state-dim=39 \
|
| 273 |
+
--data.transform.model.resize.size 240 320 \
|
| 274 |
+
--data.transform.model.center_crop.size 240 320 \
|
| 275 |
+
--data.transform.model.no-img-aug \
|
| 276 |
+
--data.transform.model.text-only \
|
| 277 |
+
--data.transform.model.audio-only \
|
| 278 |
+
--data.transform.model.video-only \
|
| 279 |
+
--data.transform.model.video-size-pixels=$VIDEO_SIZE_PIXELS \
|
| 280 |
+
--model.model_name_or_path=$PSI_PRETRAIN_BACKBONE_PATH \
|
| 281 |
+
--model.noise-scheduler=flow \
|
| 282 |
+
--model.n_conditions=0 \
|
| 283 |
+
--model.action-chunk-size=$ACTION_CHUNK_SIZE \
|
| 284 |
+
--model.action-dim=39 \
|
| 285 |
+
--model.action-exec-horizon=$ACTION_EXEC_HORIZON \
|
| 286 |
+
--model.observation-horizon=$STATE_HORIZON \
|
| 287 |
+
--model.odim=39 \
|
| 288 |
+
--model.view_feature_dim=2048 \
|
| 289 |
+
--model.use-future-progress-embedding \
|
| 290 |
+
--model.future-progress-dim=$PROGRESS_DIM \
|
| 291 |
+
--model.tune-vlm \
|
| 292 |
+
--model.lang-backbone-lr=1e-5 \
|
| 293 |
+
--model.no-use_film \
|
| 294 |
+
--model.no-combined_temb \
|
| 295 |
+
--model.rtc \
|
| 296 |
+
--model.max-delay=11 \
|
| 297 |
+
--train.max-grad-value=1 \
|
| 298 |
+
--train.loss-group-weight.joint-abs=1.0 \
|
| 299 |
+
--train.loss-group-weight.root-delta-xy=3.0 \
|
| 300 |
+
--train.loss-group-weight.root-rot5-delta=10.0 \
|
| 301 |
+
--train.loss-group-weight.root-height=15.0 \
|
| 302 |
+
--train.loss-group-weight.foot-contact=0.1
|
| 303 |
+
"
|
| 304 |
+
torchrun --nproc_per_node=$NPROC_PER_NODE --master_port=29500 scripts/train.py \
|
| 305 |
+
$args "$@"
|
posttrain-v1-psi0-text-ckpt-continue-video-clip-tune-vlm.sh
ADDED
|
@@ -0,0 +1,290 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
pkill -f monitor.py
|
| 2 |
+
|
| 3 |
+
cd /cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/Psi0-yth-v18
|
| 4 |
+
export PYTHONPATH=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/Psi0-yth-v18/src
|
| 5 |
+
export CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7
|
| 6 |
+
export PSI_OUTPUT_DIR=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/Psi0-yth-v18/runs
|
| 7 |
+
export PSI_LOG_REPORT_TO=wandb
|
| 8 |
+
|
| 9 |
+
source .venv-psi/bin/activate
|
| 10 |
+
[ -f .env ] && source .env
|
| 11 |
+
|
| 12 |
+
export WANDB_API_KEY="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"
|
| 13 |
+
|
| 14 |
+
export OMP_NUM_THREADS=8
|
| 15 |
+
export TF_CPP_MIN_LOG_LEVEL=3
|
| 16 |
+
|
| 17 |
+
# Keep JSON sources readable in this script, but pass compact JSON to tyro.
|
| 18 |
+
compact_json() {
|
| 19 |
+
python -c 'import json, sys; print(json.dumps(json.load(sys.stdin), separators=(",", ":")))'
|
| 20 |
+
}
|
| 21 |
+
|
| 22 |
+
PSI_SEED_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/Psi0/dataset/seed_all
|
| 23 |
+
PSI_SEED_METADATA=metadata/seed_metadata_v004.csv
|
| 24 |
+
|
| 25 |
+
PSI_DANCE_MOTION_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/GR00T-WholeBodyControl/gear_sonic_deploy/reference/converted/psi0_dance/gvhmr
|
| 26 |
+
PSI_DANCE_AUDIO_VIDEO_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/dance_1723_video_psi0/dance_1723_video_10s
|
| 27 |
+
PSI_DANCE_VIDEO_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/dance_1723_video_psi0/dance_1723_video_10s
|
| 28 |
+
PSI_DANCE_VIDEO_EMBED_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/dance_1723_video_psi0/dance_1723_video_10s_embeds
|
| 29 |
+
PSI_VIDEO_DEMO_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/videogen/motion_example_0506_demo
|
| 30 |
+
PSI_QWEN3_TTS_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/qwen3-tts
|
| 31 |
+
PSI_VIDEO_CLIP_DATASET_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/videogen/psi0_video_clip_50f_rand8_full
|
| 32 |
+
PSI_DANCE_VIDEO_CLIP_ROOT=$PSI_VIDEO_CLIP_DATASET_ROOT/dance_1723_video_10s
|
| 33 |
+
PSI_MOTION_EXAMPLE_VIDEO_CLIP_ROOT=$PSI_VIDEO_CLIP_DATASET_ROOT/motion_example_0506_demo
|
| 34 |
+
PSI_MOTION_EXAMPLE_VIDEO_EMBED_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/videogen/motion_example_0506_demo_video_embeds
|
| 35 |
+
|
| 36 |
+
if [ -z "$PSI_AUDIO_SOURCES_JSON" ]; then
|
| 37 |
+
PSI_AUDIO_SOURCES_JSON=$(compact_json <<JSON
|
| 38 |
+
[
|
| 39 |
+
{
|
| 40 |
+
"name": "psi0_dance_audio",
|
| 41 |
+
"root_dir": "$PSI_DANCE_MOTION_ROOT",
|
| 42 |
+
"video_dir": "$PSI_DANCE_AUDIO_VIDEO_ROOT",
|
| 43 |
+
"dataset_name": "seed-audio-g1"
|
| 44 |
+
}
|
| 45 |
+
]
|
| 46 |
+
JSON
|
| 47 |
+
)
|
| 48 |
+
fi
|
| 49 |
+
|
| 50 |
+
if [ -z "$PSI_AUDIO_COMMAND_SOURCES_JSON" ]; then
|
| 51 |
+
PSI_AUDIO_COMMAND_SOURCES_JSON=$(compact_json <<JSON
|
| 52 |
+
[
|
| 53 |
+
{
|
| 54 |
+
"name": "qwen3_tts_seed_audio_command",
|
| 55 |
+
"root_dir": "$PSI_SEED_ROOT",
|
| 56 |
+
"dataset_root": "$PSI_QWEN3_TTS_ROOT",
|
| 57 |
+
"metadata_jsonl": "metadata.jsonl",
|
| 58 |
+
"motion_dir_key": "filename",
|
| 59 |
+
"audio_path_key": "audio_path",
|
| 60 |
+
"instruction_key": "content_short_description",
|
| 61 |
+
"motion_metadata_csv": "$PSI_SEED_METADATA",
|
| 62 |
+
"motion_metadata_key": "filename",
|
| 63 |
+
"motion_metadata_path_key": "move_g1_path",
|
| 64 |
+
"audio_file_extensions": ".wav",
|
| 65 |
+
"audio_task": "command",
|
| 66 |
+
"dataset_name": "seed-audio-command-g1"
|
| 67 |
+
}
|
| 68 |
+
]
|
| 69 |
+
JSON
|
| 70 |
+
)
|
| 71 |
+
fi
|
| 72 |
+
|
| 73 |
+
if [ -z "$PSI_VIDEO_SOURCES_JSON" ]; then
|
| 74 |
+
PSI_VIDEO_SOURCES_JSON=$(compact_json <<JSON
|
| 75 |
+
[
|
| 76 |
+
{
|
| 77 |
+
"name": "psi0_dance_gvhmr",
|
| 78 |
+
"root_dir": "$PSI_DANCE_MOTION_ROOT",
|
| 79 |
+
"video_dir": "$PSI_DANCE_VIDEO_ROOT",
|
| 80 |
+
"video_embed_dir": "$PSI_DANCE_VIDEO_EMBED_ROOT",
|
| 81 |
+
"dataset_name": "seed-video-g1"
|
| 82 |
+
},
|
| 83 |
+
{
|
| 84 |
+
"name": "motion_example_0506_demo",
|
| 85 |
+
"dataset_root": "$PSI_VIDEO_DEMO_ROOT",
|
| 86 |
+
"metadata_jsonl": "metadata.jsonl",
|
| 87 |
+
"motion_dir_key": "converted_path",
|
| 88 |
+
"video_path_key": "video_path",
|
| 89 |
+
"instruction_key": "video_caption",
|
| 90 |
+
"video_embed_dir": "$PSI_MOTION_EXAMPLE_VIDEO_EMBED_ROOT",
|
| 91 |
+
"dataset_name": "motion-example-video-g1"
|
| 92 |
+
}
|
| 93 |
+
]
|
| 94 |
+
JSON
|
| 95 |
+
)
|
| 96 |
+
fi
|
| 97 |
+
|
| 98 |
+
if [ -z "$PSI_VIDEO_CLIP_SOURCES_JSON" ]; then
|
| 99 |
+
PSI_VIDEO_CLIP_SOURCES_JSON=$(compact_json <<JSON
|
| 100 |
+
[
|
| 101 |
+
{
|
| 102 |
+
"name": "dance_1723_video_10s_clip50",
|
| 103 |
+
"dataset_root": "$PSI_DANCE_VIDEO_CLIP_ROOT",
|
| 104 |
+
"metadata_jsonl": "metadata.jsonl",
|
| 105 |
+
"motion_dir_key": "converted_path",
|
| 106 |
+
"video_path_key": "video_path",
|
| 107 |
+
"instruction_key": "video_caption",
|
| 108 |
+
"video_embed_dir": "$PSI_DANCE_VIDEO_CLIP_ROOT/embeds",
|
| 109 |
+
"dataset_name": "dance-video-clip-g1"
|
| 110 |
+
},
|
| 111 |
+
{
|
| 112 |
+
"name": "motion_example_0506_demo_clip50",
|
| 113 |
+
"dataset_root": "$PSI_MOTION_EXAMPLE_VIDEO_CLIP_ROOT",
|
| 114 |
+
"metadata_jsonl": "metadata.jsonl",
|
| 115 |
+
"motion_dir_key": "converted_path",
|
| 116 |
+
"video_path_key": "video_path",
|
| 117 |
+
"instruction_key": "video_caption",
|
| 118 |
+
"video_embed_dir": "$PSI_MOTION_EXAMPLE_VIDEO_CLIP_ROOT/embeds",
|
| 119 |
+
"dataset_name": "motion-example-video-clip-g1"
|
| 120 |
+
}
|
| 121 |
+
]
|
| 122 |
+
JSON
|
| 123 |
+
)
|
| 124 |
+
fi
|
| 125 |
+
|
| 126 |
+
PSI_RESUME_TEXT_CHECKPOINT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/Psi0-yth-v18/runs/posttrain_v1_trimodal/posttrain.sg.flow1000.cwml.lr1.0e-04.b256.gpus8.2605310619/checkpoints/ckpt_280000
|
| 127 |
+
|
| 128 |
+
PSI_TEXT_TASK_WEIGHT=0.5
|
| 129 |
+
PSI_AUDIO_TEXT_TASK_WEIGHT=0
|
| 130 |
+
PSI_AUDIO_ONLY_TASK_WEIGHT=0
|
| 131 |
+
PSI_AUDIO_COMMAND_TEXT_TASK_WEIGHT=0
|
| 132 |
+
PSI_AUDIO_COMMAND_ONLY_TASK_WEIGHT=0
|
| 133 |
+
PSI_VIDEO_TEXT_TASK_WEIGHT=0
|
| 134 |
+
PSI_VIDEO_ONLY_TASK_WEIGHT=0
|
| 135 |
+
PSI_VIDEO_CLIP_TEXT_TASK_WEIGHT=0.25
|
| 136 |
+
PSI_VIDEO_CLIP_ONLY_TASK_WEIGHT=0.25
|
| 137 |
+
|
| 138 |
+
PSI_PRETRAIN_BACKBONE_PATH=/cpfs01/VideoGen/public_ckpt/Qwen/Qwen2.5-Omni-3B
|
| 139 |
+
NUM_PAST_FRAMES=0
|
| 140 |
+
STATE_HORIZON=$((NUM_PAST_FRAMES + 1))
|
| 141 |
+
ACTION_CHUNK_SIZE=30
|
| 142 |
+
ACTION_EXEC_HORIZON=20
|
| 143 |
+
PROGRESS_DIM=5
|
| 144 |
+
VIDEO_SAMPLE_FPS=1.0
|
| 145 |
+
VIDEO_CLIP_SAMPLE_FPS=8.0
|
| 146 |
+
VIDEO_SIZE_PIXELS=50176
|
| 147 |
+
VIDEO_DECODER=ffmpeg
|
| 148 |
+
VIDEO_CACHE_SIZE=64
|
| 149 |
+
DATALOADER_NUM_WORKERS=4
|
| 150 |
+
VAL_DATALOADER_NUM_WORKERS=2
|
| 151 |
+
|
| 152 |
+
if [ ! -f "$PSI_RESUME_TEXT_CHECKPOINT/model.safetensors" ] || [ ! -f "$PSI_RESUME_TEXT_CHECKPOINT/optimizer.bin" ] || [ ! -f "$PSI_RESUME_TEXT_CHECKPOINT/scheduler.bin" ]; then
|
| 153 |
+
echo "ERROR: resume checkpoint is not a full training-state checkpoint: $PSI_RESUME_TEXT_CHECKPOINT" >&2
|
| 154 |
+
exit 1
|
| 155 |
+
fi
|
| 156 |
+
|
| 157 |
+
NPROC_PER_NODE=$(echo $CUDA_VISIBLE_DEVICES | tr ',' '\n' | wc -l)
|
| 158 |
+
|
| 159 |
+
echo "Training with $NPROC_PER_NODE GPU(s): $CUDA_VISIBLE_DEVICES"
|
| 160 |
+
echo "Text root: $PSI_SEED_ROOT"
|
| 161 |
+
echo "Dance motion root: $PSI_DANCE_MOTION_ROOT"
|
| 162 |
+
echo "Dance audio/video root: $PSI_DANCE_AUDIO_VIDEO_ROOT"
|
| 163 |
+
echo "Dance video root: $PSI_DANCE_VIDEO_ROOT"
|
| 164 |
+
echo "Dance video embed root: $PSI_DANCE_VIDEO_EMBED_ROOT"
|
| 165 |
+
echo "Qwen3-TTS acdudio command root: $PSI_QWEN3_TTS_ROOT"
|
| 166 |
+
echo "Audio sources: $PSI_AUDIO_SOURCES_JSON"
|
| 167 |
+
echo "Audio command sources: $PSI_AUDIO_COMMAND_SOURCES_JSON"
|
| 168 |
+
echo "Video sources: $PSI_VIDEO_SOURCES_JSON"
|
| 169 |
+
echo "Video clip sources: $PSI_VIDEO_CLIP_SOURCES_JSON"
|
| 170 |
+
echo "Task weights: text=$PSI_TEXT_TASK_WEIGHT audio_text=$PSI_AUDIO_TEXT_TASK_WEIGHT audio_only=$PSI_AUDIO_ONLY_TASK_WEIGHT audio_command_text=$PSI_AUDIO_COMMAND_TEXT_TASK_WEIGHT audio_command_only=$PSI_AUDIO_COMMAND_ONLY_TASK_WEIGHT video_text=$PSI_VIDEO_TEXT_TASK_WEIGHT video_only=$PSI_VIDEO_ONLY_TASK_WEIGHT video_clip_text=$PSI_VIDEO_CLIP_TEXT_TASK_WEIGHT video_clip_only=$PSI_VIDEO_CLIP_ONLY_TASK_WEIGHT"
|
| 171 |
+
echo "Resume full training state from: $PSI_RESUME_TEXT_CHECKPOINT"
|
| 172 |
+
echo "TextOp 39D action/state enabled"
|
| 173 |
+
echo "Future progress embedding: enabled, dim $PROGRESS_DIM"
|
| 174 |
+
echo "Backbone path: $PSI_PRETRAIN_BACKBONE_PATH"
|
| 175 |
+
echo "Output dir: $PSI_OUTPUT_DIR"
|
| 176 |
+
|
| 177 |
+
args="posttrain_v1_seed_text_audio_video_mix_psi0_config \
|
| 178 |
+
--seed=292285 \
|
| 179 |
+
--exp=posttrain \
|
| 180 |
+
--timestamp=$(date +"%y%m%d%H%M") \
|
| 181 |
+
--train.name=posttrain_v1_trimodal \
|
| 182 |
+
--train.data_parallel=ddp \
|
| 183 |
+
--train.mixed_precision=bf16 \
|
| 184 |
+
--train.train_batch_size=32 \
|
| 185 |
+
--train.val_batch_size=64 \
|
| 186 |
+
--train.num-workers=$DATALOADER_NUM_WORKERS \
|
| 187 |
+
--train.val-num-workers=$VAL_DATALOADER_NUM_WORKERS \
|
| 188 |
+
--train.resume_from_checkpoint=$PSI_RESUME_TEXT_CHECKPOINT \
|
| 189 |
+
--train.resume-load-full-state \
|
| 190 |
+
--train.max_checkpoints_to_keep=1 \
|
| 191 |
+
--train.gradient_accumulation_steps=1 \
|
| 192 |
+
--train.learning_rate=1e-4 \
|
| 193 |
+
--train.max_training_steps=500000 \
|
| 194 |
+
--train.warmup_ratio=None \
|
| 195 |
+
--train.warmup_steps=1000 \
|
| 196 |
+
--train.checkpointing_steps=1000 \
|
| 197 |
+
--train.validation_steps=1000 \
|
| 198 |
+
--train.val_num_batches=20 \
|
| 199 |
+
--train.max_grad_norm=1.0 \
|
| 200 |
+
--train.lr_scheduler_type=cosine_with_min_lr \
|
| 201 |
+
--train.lr_scheduler_kwargs.betas 0.9 0.999 \
|
| 202 |
+
--train.lr_scheduler_kwargs.weight_decay=0 \
|
| 203 |
+
--train.lr_scheduler_kwargs.eps=1e-8 \
|
| 204 |
+
--log.report_to=$PSI_LOG_REPORT_TO \
|
| 205 |
+
--train.output_dir=$PSI_OUTPUT_DIR \
|
| 206 |
+
--data.text.root-dir=$PSI_SEED_ROOT \
|
| 207 |
+
--data.text.metadata-csv=$PSI_SEED_METADATA \
|
| 208 |
+
--data.text-ratio=$PSI_TEXT_TASK_WEIGHT \
|
| 209 |
+
--data.audio.sources-json=$PSI_AUDIO_SOURCES_JSON \
|
| 210 |
+
--data.audio-ratio=$PSI_AUDIO_TEXT_TASK_WEIGHT \
|
| 211 |
+
--data.audio-only-ratio=$PSI_AUDIO_ONLY_TASK_WEIGHT \
|
| 212 |
+
--data.audio.sample-stride=1 \
|
| 213 |
+
--data.audio.motion-fps=50 \
|
| 214 |
+
--data.audio.audio-sampling-rate=16000 \
|
| 215 |
+
--data.audio.audio-window-seconds=1.0 \
|
| 216 |
+
--data.audio.audio-left-context-seconds=0.0 \
|
| 217 |
+
--data.audio.full-audio \
|
| 218 |
+
--data.audio-command.sources-json=$PSI_AUDIO_COMMAND_SOURCES_JSON \
|
| 219 |
+
--data.audio-command-ratio=$PSI_AUDIO_COMMAND_TEXT_TASK_WEIGHT \
|
| 220 |
+
--data.audio-command-only-ratio=$PSI_AUDIO_COMMAND_ONLY_TASK_WEIGHT \
|
| 221 |
+
--data.audio-command.sample-stride=1 \
|
| 222 |
+
--data.audio-command.motion-fps=50 \
|
| 223 |
+
--data.audio-command.audio-sampling-rate=16000 \
|
| 224 |
+
--data.audio-command.audio-window-seconds=1.0 \
|
| 225 |
+
--data.audio-command.audio-left-context-seconds=0.0 \
|
| 226 |
+
--data.audio-command.full-audio \
|
| 227 |
+
--data.video-ratio=$PSI_VIDEO_TEXT_TASK_WEIGHT \
|
| 228 |
+
--data.video-only-ratio=$PSI_VIDEO_ONLY_TASK_WEIGHT \
|
| 229 |
+
--data.video.sample-stride=1 \
|
| 230 |
+
--data.video.motion-fps=50 \
|
| 231 |
+
--data.video.video-sample-fps=$VIDEO_SAMPLE_FPS \
|
| 232 |
+
--data.video.video-size-pixels=$VIDEO_SIZE_PIXELS \
|
| 233 |
+
--data.video.video-decoder=$VIDEO_DECODER \
|
| 234 |
+
--data.video.video-cache-size=$VIDEO_CACHE_SIZE \
|
| 235 |
+
--data.video.sources-json=$PSI_VIDEO_SOURCES_JSON \
|
| 236 |
+
--data.video-clip-text-ratio=$PSI_VIDEO_CLIP_TEXT_TASK_WEIGHT \
|
| 237 |
+
--data.video-clip-only-ratio=$PSI_VIDEO_CLIP_ONLY_TASK_WEIGHT \
|
| 238 |
+
--data.video-clip.sources-json=$PSI_VIDEO_CLIP_SOURCES_JSON \
|
| 239 |
+
--data.video-clip.sample-stride=1 \
|
| 240 |
+
--data.video-clip.motion-fps=50 \
|
| 241 |
+
--data.video-clip.video-sample-fps=$VIDEO_CLIP_SAMPLE_FPS \
|
| 242 |
+
--data.video-clip.video-size-pixels=$VIDEO_SIZE_PIXELS \
|
| 243 |
+
--data.video-clip.video-decoder=$VIDEO_DECODER \
|
| 244 |
+
--data.video-clip.video-cache-size=$VIDEO_CACHE_SIZE \
|
| 245 |
+
--data.video-clip.action-exec-horizon=$ACTION_EXEC_HORIZON \
|
| 246 |
+
--data.sampler=batch_mixture \
|
| 247 |
+
--data.val-task=text,video_clip,video_clip_only \
|
| 248 |
+
--data.transform.repack.num-past-frames=$NUM_PAST_FRAMES \
|
| 249 |
+
--data.transform.repack.textop-actions \
|
| 250 |
+
--data.transform.repack.action-chunk-size=$ACTION_CHUNK_SIZE \
|
| 251 |
+
--data.transform.repack.pad-action-dim=39 \
|
| 252 |
+
--data.transform.repack.pad-state-dim=39 \
|
| 253 |
+
--data.transform.field.action_norm_type=bounds_q99 \
|
| 254 |
+
--data.transform.field.stat-path=assets/stats/seed_all_g1_motion39_textop_stats.json \
|
| 255 |
+
--data.transform.field.normalize-state \
|
| 256 |
+
--data.transform.field.pad-action-dim=39 \
|
| 257 |
+
--data.transform.field.pad-state-dim=39 \
|
| 258 |
+
--data.transform.model.resize.size 240 320 \
|
| 259 |
+
--data.transform.model.center_crop.size 240 320 \
|
| 260 |
+
--data.transform.model.no-img-aug \
|
| 261 |
+
--data.transform.model.text-only \
|
| 262 |
+
--data.transform.model.audio-only \
|
| 263 |
+
--data.transform.model.video-only \
|
| 264 |
+
--data.transform.model.video-size-pixels=$VIDEO_SIZE_PIXELS \
|
| 265 |
+
--model.model_name_or_path=$PSI_PRETRAIN_BACKBONE_PATH \
|
| 266 |
+
--model.noise-scheduler=flow \
|
| 267 |
+
--model.n_conditions=0 \
|
| 268 |
+
--model.action-chunk-size=$ACTION_CHUNK_SIZE \
|
| 269 |
+
--model.action-dim=39 \
|
| 270 |
+
--model.action-exec-horizon=$ACTION_EXEC_HORIZON \
|
| 271 |
+
--model.observation-horizon=$STATE_HORIZON \
|
| 272 |
+
--model.odim=39 \
|
| 273 |
+
--model.view_feature_dim=2048 \
|
| 274 |
+
--model.use-future-progress-embedding \
|
| 275 |
+
--model.future-progress-dim=$PROGRESS_DIM \
|
| 276 |
+
--model.tune-vlm \
|
| 277 |
+
--model.lang-backbone-lr=1e-5 \
|
| 278 |
+
--model.no-use_film \
|
| 279 |
+
--model.no-combined_temb \
|
| 280 |
+
--model.rtc \
|
| 281 |
+
--model.max-delay=11 \
|
| 282 |
+
--train.max-grad-value=1 \
|
| 283 |
+
--train.loss-group-weight.joint-abs=1.0 \
|
| 284 |
+
--train.loss-group-weight.root-delta-xy=3.0 \
|
| 285 |
+
--train.loss-group-weight.root-rot5-delta=10.0 \
|
| 286 |
+
--train.loss-group-weight.root-height=15.0 \
|
| 287 |
+
--train.loss-group-weight.foot-contact=0.1
|
| 288 |
+
"
|
| 289 |
+
torchrun --nproc_per_node=$NPROC_PER_NODE --master_port=29500 scripts/train.py \
|
| 290 |
+
$args "$@"
|
prebuild-multimodal-records-cache.sh
ADDED
|
@@ -0,0 +1,280 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
cd /cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/Psi0-yth-v21
|
| 2 |
+
export PYTHONPATH=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/Psi0-yth-v21/src
|
| 3 |
+
|
| 4 |
+
source /cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/Psi0-yth-v15/.venv-psi/bin/activate
|
| 5 |
+
[ -f .env ] && source .env
|
| 6 |
+
|
| 7 |
+
export OMP_NUM_THREADS=${OMP_NUM_THREADS:-8}
|
| 8 |
+
export TF_CPP_MIN_LOG_LEVEL=${TF_CPP_MIN_LOG_LEVEL:-3}
|
| 9 |
+
|
| 10 |
+
compact_json() {
|
| 11 |
+
python -c 'import json, sys; print(json.dumps(json.load(sys.stdin), separators=(",", ":")))'
|
| 12 |
+
}
|
| 13 |
+
|
| 14 |
+
PSI_SEED_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/Psi0/dataset/seed_all
|
| 15 |
+
PSI_SEED_METADATA=metadata/seed_metadata_v004.csv
|
| 16 |
+
PSI_TEXT_RECORDS_CACHE_DIR=${PSI_TEXT_RECORDS_CACHE_DIR:-$PSI_SEED_ROOT/.index_cache_multimodal_yth}
|
| 17 |
+
|
| 18 |
+
PSI_DANCE_MOTION_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/GR00T-WholeBodyControl/gear_sonic_deploy/reference/converted/psi0_dance/gvhmr
|
| 19 |
+
PSI_DANCE_AUDIO_VIDEO_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/dance_1723_video_psi0/dance_1723_video_10s
|
| 20 |
+
PSI_DANCE_VIDEO_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/dance_1723_video_psi0/dance_1723_video_10s
|
| 21 |
+
PSI_DANCE_VIDEO_EMBED_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/dance_1723_video_psi0/dance_1723_video_10s_embeds
|
| 22 |
+
PSI_VIDEO_DEMO_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/videogen/motion_example_0506_demo
|
| 23 |
+
PSI_2264_WITH_AUDIO_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/videogen/2264_with_audio
|
| 24 |
+
PSI_QWEN3_TTS_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/qwen3-tts
|
| 25 |
+
PSI_VIDEO_CLIP_DATASET_ROOT=${PSI_VIDEO_CLIP_DATASET_ROOT:-/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/videogen/psi0_video_clip_50f_dense_stride1_direct_full}
|
| 26 |
+
PSI_DANCE_VIDEO_CLIP_ROOT=$PSI_VIDEO_CLIP_DATASET_ROOT/dance_1723_video_10s
|
| 27 |
+
PSI_MOTION_EXAMPLE_VIDEO_CLIP_ROOT=$PSI_VIDEO_CLIP_DATASET_ROOT/motion_example_0506_demo
|
| 28 |
+
PSI_2264_WITH_AUDIO_VIDEO_CLIP_ROOT=$PSI_VIDEO_CLIP_DATASET_ROOT/2264_with_audio
|
| 29 |
+
PSI_DANCE_VIDEO_CLIP_EMBED_ROOT=${PSI_DANCE_VIDEO_CLIP_EMBED_ROOT:-$PSI_DANCE_VIDEO_CLIP_ROOT/embeds}
|
| 30 |
+
PSI_MOTION_EXAMPLE_VIDEO_CLIP_EMBED_ROOT=${PSI_MOTION_EXAMPLE_VIDEO_CLIP_EMBED_ROOT:-$PSI_MOTION_EXAMPLE_VIDEO_CLIP_ROOT/embeds}
|
| 31 |
+
PSI_2264_WITH_AUDIO_VIDEO_CLIP_EMBED_ROOT=${PSI_2264_WITH_AUDIO_VIDEO_CLIP_EMBED_ROOT:-$PSI_2264_WITH_AUDIO_VIDEO_CLIP_ROOT/embeds}
|
| 32 |
+
PSI_MOTION_EXAMPLE_VIDEO_EMBED_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/videogen/motion_example_0506_demo_video_embeds
|
| 33 |
+
PSI_AUDIO_RECORDS_CACHE_DIR=${PSI_AUDIO_RECORDS_CACHE_DIR:-$PSI_2264_WITH_AUDIO_ROOT/.index_cache_multimodal_yth}
|
| 34 |
+
PSI_AUDIO_COMMAND_RECORDS_CACHE_DIR=${PSI_AUDIO_COMMAND_RECORDS_CACHE_DIR:-$PSI_QWEN3_TTS_ROOT/.index_cache_multimodal_yth}
|
| 35 |
+
PSI_DANCE_AUDIO_NPY_ROOT=${PSI_DANCE_AUDIO_NPY_ROOT:-$PSI_DANCE_AUDIO_VIDEO_ROOT/audio_npy_yth}
|
| 36 |
+
PSI_DANCE_AUDIO_NPY_MANIFEST=${PSI_DANCE_AUDIO_NPY_MANIFEST:-$PSI_DANCE_AUDIO_VIDEO_ROOT/metadata_audio_npy.jsonl}
|
| 37 |
+
PSI_2264_AUDIO_NPY_MANIFEST=${PSI_2264_AUDIO_NPY_MANIFEST:-$PSI_2264_WITH_AUDIO_ROOT/metadata_audio_npy.jsonl}
|
| 38 |
+
PSI_QWEN3_TTS_AUDIO_NPY_ROOT=${PSI_QWEN3_TTS_AUDIO_NPY_ROOT:-$PSI_QWEN3_TTS_ROOT/audio_npy_yth}
|
| 39 |
+
PSI_QWEN3_TTS_AUDIO_NPY_MANIFEST=${PSI_QWEN3_TTS_AUDIO_NPY_MANIFEST:-$PSI_QWEN3_TTS_ROOT/metadata_audio_npy.jsonl}
|
| 40 |
+
|
| 41 |
+
if [ -f "$PSI_2264_AUDIO_NPY_MANIFEST" ]; then
|
| 42 |
+
PSI_2264_AUDIO_METADATA_JSONL=$PSI_2264_AUDIO_NPY_MANIFEST
|
| 43 |
+
PSI_2264_AUDIO_PATH_KEY=audio_path
|
| 44 |
+
else
|
| 45 |
+
PSI_2264_AUDIO_METADATA_JSONL=metadata_convert.jsonl
|
| 46 |
+
PSI_2264_AUDIO_PATH_KEY=video_path
|
| 47 |
+
fi
|
| 48 |
+
|
| 49 |
+
if [ -f "$PSI_DANCE_AUDIO_NPY_MANIFEST" ]; then
|
| 50 |
+
PSI_DANCE_AUDIO_SOURCE_JSON=$(cat <<JSON
|
| 51 |
+
{
|
| 52 |
+
"name": "psi0_dance_audio",
|
| 53 |
+
"dataset_root": "$PSI_DANCE_AUDIO_VIDEO_ROOT",
|
| 54 |
+
"metadata_jsonl": "$PSI_DANCE_AUDIO_NPY_MANIFEST",
|
| 55 |
+
"motion_dir_key": "converted_path",
|
| 56 |
+
"audio_path_key": "audio_path",
|
| 57 |
+
"records_cache_dir": "$PSI_AUDIO_RECORDS_CACHE_DIR",
|
| 58 |
+
"dataset_name": "seed-audio-g1"
|
| 59 |
+
}
|
| 60 |
+
JSON
|
| 61 |
+
)
|
| 62 |
+
else
|
| 63 |
+
PSI_DANCE_AUDIO_SOURCE_JSON=$(cat <<JSON
|
| 64 |
+
{
|
| 65 |
+
"name": "psi0_dance_audio",
|
| 66 |
+
"root_dir": "$PSI_DANCE_MOTION_ROOT",
|
| 67 |
+
"video_dir": "$PSI_DANCE_AUDIO_VIDEO_ROOT",
|
| 68 |
+
"records_cache_dir": "$PSI_AUDIO_RECORDS_CACHE_DIR",
|
| 69 |
+
"dataset_name": "seed-audio-g1"
|
| 70 |
+
}
|
| 71 |
+
JSON
|
| 72 |
+
)
|
| 73 |
+
fi
|
| 74 |
+
|
| 75 |
+
if [ -z "$PSI_AUDIO_SOURCES_JSON" ]; then
|
| 76 |
+
PSI_AUDIO_SOURCES_JSON=$(compact_json <<JSON
|
| 77 |
+
[
|
| 78 |
+
$PSI_DANCE_AUDIO_SOURCE_JSON,
|
| 79 |
+
{
|
| 80 |
+
"name": "2264_with_audio",
|
| 81 |
+
"dataset_root": "$PSI_2264_WITH_AUDIO_ROOT",
|
| 82 |
+
"metadata_jsonl": "$PSI_2264_AUDIO_METADATA_JSONL",
|
| 83 |
+
"motion_dir_key": "converted_path",
|
| 84 |
+
"audio_path_key": "$PSI_2264_AUDIO_PATH_KEY",
|
| 85 |
+
"records_cache_dir": "$PSI_AUDIO_RECORDS_CACHE_DIR",
|
| 86 |
+
"dataset_name": "2264-with-audio-g1"
|
| 87 |
+
}
|
| 88 |
+
]
|
| 89 |
+
JSON
|
| 90 |
+
)
|
| 91 |
+
fi
|
| 92 |
+
|
| 93 |
+
if [ -f "$PSI_QWEN3_TTS_AUDIO_NPY_MANIFEST" ]; then
|
| 94 |
+
PSI_AUDIO_COMMAND_SOURCE_JSON=$(cat <<JSON
|
| 95 |
+
{
|
| 96 |
+
"name": "qwen3_tts_seed_audio_command",
|
| 97 |
+
"root_dir": "$PSI_SEED_ROOT",
|
| 98 |
+
"dataset_root": "$PSI_QWEN3_TTS_ROOT",
|
| 99 |
+
"metadata_jsonl": "$PSI_QWEN3_TTS_AUDIO_NPY_MANIFEST",
|
| 100 |
+
"motion_dir_key": "filename",
|
| 101 |
+
"audio_path_key": "audio_path",
|
| 102 |
+
"instruction_key": "content_short_description",
|
| 103 |
+
"motion_metadata_csv": "$PSI_SEED_METADATA",
|
| 104 |
+
"motion_metadata_key": "filename",
|
| 105 |
+
"motion_metadata_path_key": "move_g1_path",
|
| 106 |
+
"audio_file_extensions": ".npy",
|
| 107 |
+
"records_cache_dir": "$PSI_AUDIO_COMMAND_RECORDS_CACHE_DIR",
|
| 108 |
+
"audio_task": "command",
|
| 109 |
+
"dataset_name": "seed-audio-command-g1"
|
| 110 |
+
}
|
| 111 |
+
JSON
|
| 112 |
+
)
|
| 113 |
+
else
|
| 114 |
+
PSI_AUDIO_COMMAND_SOURCE_JSON=$(cat <<JSON
|
| 115 |
+
{
|
| 116 |
+
"name": "qwen3_tts_seed_audio_command",
|
| 117 |
+
"root_dir": "$PSI_SEED_ROOT",
|
| 118 |
+
"dataset_root": "$PSI_QWEN3_TTS_ROOT",
|
| 119 |
+
"metadata_jsonl": "metadata.jsonl",
|
| 120 |
+
"motion_dir_key": "filename",
|
| 121 |
+
"audio_path_key": "audio_path",
|
| 122 |
+
"instruction_key": "content_short_description",
|
| 123 |
+
"motion_metadata_csv": "$PSI_SEED_METADATA",
|
| 124 |
+
"motion_metadata_key": "filename",
|
| 125 |
+
"motion_metadata_path_key": "move_g1_path",
|
| 126 |
+
"audio_file_extensions": ".wav",
|
| 127 |
+
"records_cache_dir": "$PSI_AUDIO_COMMAND_RECORDS_CACHE_DIR",
|
| 128 |
+
"audio_task": "command",
|
| 129 |
+
"dataset_name": "seed-audio-command-g1"
|
| 130 |
+
}
|
| 131 |
+
JSON
|
| 132 |
+
)
|
| 133 |
+
fi
|
| 134 |
+
|
| 135 |
+
if [ -z "$PSI_AUDIO_COMMAND_SOURCES_JSON" ]; then
|
| 136 |
+
PSI_AUDIO_COMMAND_SOURCES_JSON=$(compact_json <<JSON
|
| 137 |
+
[
|
| 138 |
+
$PSI_AUDIO_COMMAND_SOURCE_JSON
|
| 139 |
+
]
|
| 140 |
+
JSON
|
| 141 |
+
)
|
| 142 |
+
fi
|
| 143 |
+
|
| 144 |
+
if [ -z "$PSI_VIDEO_SOURCES_JSON" ]; then
|
| 145 |
+
PSI_VIDEO_SOURCES_JSON=$(compact_json <<JSON
|
| 146 |
+
[
|
| 147 |
+
{
|
| 148 |
+
"name": "psi0_dance_gvhmr",
|
| 149 |
+
"root_dir": "$PSI_DANCE_MOTION_ROOT",
|
| 150 |
+
"video_dir": "$PSI_DANCE_VIDEO_ROOT",
|
| 151 |
+
"video_embed_dir": "$PSI_DANCE_VIDEO_EMBED_ROOT",
|
| 152 |
+
"dataset_name": "seed-video-g1"
|
| 153 |
+
},
|
| 154 |
+
{
|
| 155 |
+
"name": "motion_example_0506_demo",
|
| 156 |
+
"dataset_root": "$PSI_VIDEO_DEMO_ROOT",
|
| 157 |
+
"metadata_jsonl": "metadata.jsonl",
|
| 158 |
+
"motion_dir_key": "converted_path",
|
| 159 |
+
"video_path_key": "video_path",
|
| 160 |
+
"instruction_key": "video_caption",
|
| 161 |
+
"video_embed_dir": "$PSI_MOTION_EXAMPLE_VIDEO_EMBED_ROOT",
|
| 162 |
+
"dataset_name": "motion-example-video-g1"
|
| 163 |
+
}
|
| 164 |
+
]
|
| 165 |
+
JSON
|
| 166 |
+
)
|
| 167 |
+
fi
|
| 168 |
+
|
| 169 |
+
if [ -z "$PSI_VIDEO_CLIP_SOURCES_JSON" ]; then
|
| 170 |
+
PSI_VIDEO_CLIP_SOURCES_JSON=$(compact_json <<JSON
|
| 171 |
+
[
|
| 172 |
+
{
|
| 173 |
+
"name": "dance_1723_video_10s_clip50",
|
| 174 |
+
"dataset_root": "$PSI_DANCE_VIDEO_CLIP_ROOT",
|
| 175 |
+
"metadata_jsonl": "metadata.jsonl",
|
| 176 |
+
"motion_dir_key": "converted_path",
|
| 177 |
+
"video_path_key": "video_path",
|
| 178 |
+
"instruction_key": "video_caption",
|
| 179 |
+
"video_embed_dir": "$PSI_DANCE_VIDEO_CLIP_EMBED_ROOT",
|
| 180 |
+
"dataset_name": "dance-video-clip-g1"
|
| 181 |
+
},
|
| 182 |
+
{
|
| 183 |
+
"name": "motion_example_0506_demo_clip50",
|
| 184 |
+
"dataset_root": "$PSI_MOTION_EXAMPLE_VIDEO_CLIP_ROOT",
|
| 185 |
+
"metadata_jsonl": "metadata.jsonl",
|
| 186 |
+
"motion_dir_key": "converted_path",
|
| 187 |
+
"video_path_key": "video_path",
|
| 188 |
+
"instruction_key": "video_caption",
|
| 189 |
+
"video_embed_dir": "$PSI_MOTION_EXAMPLE_VIDEO_CLIP_EMBED_ROOT",
|
| 190 |
+
"dataset_name": "motion-example-video-clip-g1"
|
| 191 |
+
},
|
| 192 |
+
{
|
| 193 |
+
"name": "2264_with_audio_clip50",
|
| 194 |
+
"dataset_root": "$PSI_2264_WITH_AUDIO_VIDEO_CLIP_ROOT",
|
| 195 |
+
"metadata_jsonl": "metadata.jsonl",
|
| 196 |
+
"motion_dir_key": "converted_path",
|
| 197 |
+
"video_path_key": "video_path",
|
| 198 |
+
"instruction_key": "video_caption",
|
| 199 |
+
"video_embed_dir": "$PSI_2264_WITH_AUDIO_VIDEO_CLIP_EMBED_ROOT",
|
| 200 |
+
"dataset_name": "2264-with-audio-video-clip-g1"
|
| 201 |
+
}
|
| 202 |
+
]
|
| 203 |
+
JSON
|
| 204 |
+
)
|
| 205 |
+
fi
|
| 206 |
+
|
| 207 |
+
ACTION_CHUNK_SIZE=${ACTION_CHUNK_SIZE:-30}
|
| 208 |
+
ACTION_EXEC_HORIZON=${ACTION_EXEC_HORIZON:-20}
|
| 209 |
+
NUM_PAST_FRAMES=${NUM_PAST_FRAMES:-0}
|
| 210 |
+
VIDEO_SAMPLE_FPS=${VIDEO_SAMPLE_FPS:-1.0}
|
| 211 |
+
VIDEO_CLIP_SAMPLE_FPS=${VIDEO_CLIP_SAMPLE_FPS:-8.0}
|
| 212 |
+
VIDEO_SIZE_PIXELS=${VIDEO_SIZE_PIXELS:-50176}
|
| 213 |
+
VIDEO_DECODER=${VIDEO_DECODER:-ffmpeg}
|
| 214 |
+
VIDEO_RECORD_BUILD_WORKERS=${VIDEO_RECORD_BUILD_WORKERS:-32}
|
| 215 |
+
VIDEO_RECORD_BUILD_CHUNK_SIZE=${VIDEO_RECORD_BUILD_CHUNK_SIZE:-4096}
|
| 216 |
+
VIDEO_CLIP_TRUST_METADATA=${VIDEO_CLIP_TRUST_METADATA:-1}
|
| 217 |
+
VIDEO_CLIP_RECORDS_CACHE_DIR=${VIDEO_CLIP_RECORDS_CACHE_DIR:-$PSI_VIDEO_CLIP_DATASET_ROOT/.index_cache_multimodal_yth}
|
| 218 |
+
PREBUILD_TASKS=${PREBUILD_TASKS:-text,audio,audio_command,video_clip}
|
| 219 |
+
PREBUILD_SPLITS=${PREBUILD_SPLITS:-train,val}
|
| 220 |
+
|
| 221 |
+
if [ "$VIDEO_CLIP_TRUST_METADATA" = "1" ] || [ "$VIDEO_CLIP_TRUST_METADATA" = "true" ] || [ "$VIDEO_CLIP_TRUST_METADATA" = "True" ]; then
|
| 222 |
+
PREBUILD_VIDEO_CLIP_TRUST_METADATA_FLAG="--video-clip-trust-metadata"
|
| 223 |
+
else
|
| 224 |
+
PREBUILD_VIDEO_CLIP_TRUST_METADATA_FLAG="--no-video-clip-trust-metadata"
|
| 225 |
+
fi
|
| 226 |
+
|
| 227 |
+
echo "Prebuild tasks: $PREBUILD_TASKS"
|
| 228 |
+
echo "Prebuild splits: $PREBUILD_SPLITS"
|
| 229 |
+
echo "Text cache dir: $PSI_TEXT_RECORDS_CACHE_DIR"
|
| 230 |
+
echo "Audio cache dir: $PSI_AUDIO_RECORDS_CACHE_DIR"
|
| 231 |
+
echo "Audio command cache dir: $PSI_AUDIO_COMMAND_RECORDS_CACHE_DIR"
|
| 232 |
+
echo "Dance audio manifest: $PSI_DANCE_AUDIO_NPY_MANIFEST"
|
| 233 |
+
echo "2264 audio manifest: $PSI_2264_AUDIO_METADATA_JSONL"
|
| 234 |
+
echo "Qwen3-TTS audio manifest: $PSI_QWEN3_TTS_AUDIO_NPY_MANIFEST"
|
| 235 |
+
echo "Video clip cache dir: $VIDEO_CLIP_RECORDS_CACHE_DIR"
|
| 236 |
+
echo "Video record build workers: $VIDEO_RECORD_BUILD_WORKERS"
|
| 237 |
+
echo "Video record build chunk size: $VIDEO_RECORD_BUILD_CHUNK_SIZE"
|
| 238 |
+
|
| 239 |
+
python scripts/data/prebuild_multimodal_records_cache.py \
|
| 240 |
+
--tasks="$PREBUILD_TASKS" \
|
| 241 |
+
--splits="$PREBUILD_SPLITS" \
|
| 242 |
+
--action-chunk-size=$ACTION_CHUNK_SIZE \
|
| 243 |
+
--num-past-frames=$NUM_PAST_FRAMES \
|
| 244 |
+
--text-root-dir="$PSI_SEED_ROOT" \
|
| 245 |
+
--text-metadata-csv="$PSI_SEED_METADATA" \
|
| 246 |
+
--text-records-cache-dir="$PSI_TEXT_RECORDS_CACHE_DIR" \
|
| 247 |
+
--audio-sources-json="$PSI_AUDIO_SOURCES_JSON" \
|
| 248 |
+
--audio-records-cache-dir="$PSI_AUDIO_RECORDS_CACHE_DIR" \
|
| 249 |
+
--audio-sample-stride=1 \
|
| 250 |
+
--audio-motion-fps=50 \
|
| 251 |
+
--audio-audio-sampling-rate=16000 \
|
| 252 |
+
--audio-audio-window-seconds=1.0 \
|
| 253 |
+
--audio-audio-left-context-seconds=0.0 \
|
| 254 |
+
--audio-command-sources-json="$PSI_AUDIO_COMMAND_SOURCES_JSON" \
|
| 255 |
+
--audio-command-records-cache-dir="$PSI_AUDIO_COMMAND_RECORDS_CACHE_DIR" \
|
| 256 |
+
--audio-command-sample-stride=1 \
|
| 257 |
+
--audio-command-motion-fps=50 \
|
| 258 |
+
--audio-command-audio-sampling-rate=16000 \
|
| 259 |
+
--audio-command-audio-window-seconds=1.0 \
|
| 260 |
+
--audio-command-audio-left-context-seconds=0.0 \
|
| 261 |
+
--video-sources-json="$PSI_VIDEO_SOURCES_JSON" \
|
| 262 |
+
--video-sample-stride=1 \
|
| 263 |
+
--video-motion-fps=50 \
|
| 264 |
+
--video-video-sample-fps=$VIDEO_SAMPLE_FPS \
|
| 265 |
+
--video-video-size-pixels=$VIDEO_SIZE_PIXELS \
|
| 266 |
+
--video-video-decoder=$VIDEO_DECODER \
|
| 267 |
+
--video-record-build-workers=$VIDEO_RECORD_BUILD_WORKERS \
|
| 268 |
+
--video-record-build-chunk-size=$VIDEO_RECORD_BUILD_CHUNK_SIZE \
|
| 269 |
+
--video-clip-sources-json="$PSI_VIDEO_CLIP_SOURCES_JSON" \
|
| 270 |
+
--video-clip-records-cache-dir="$VIDEO_CLIP_RECORDS_CACHE_DIR" \
|
| 271 |
+
--video-clip-sample-stride=1 \
|
| 272 |
+
--video-clip-motion-fps=50 \
|
| 273 |
+
--video-clip-video-sample-fps=$VIDEO_CLIP_SAMPLE_FPS \
|
| 274 |
+
--video-clip-video-size-pixels=$VIDEO_SIZE_PIXELS \
|
| 275 |
+
--video-clip-video-decoder=$VIDEO_DECODER \
|
| 276 |
+
--video-clip-record-build-workers=$VIDEO_RECORD_BUILD_WORKERS \
|
| 277 |
+
--video-clip-record-build-chunk-size=$VIDEO_RECORD_BUILD_CHUNK_SIZE \
|
| 278 |
+
$PREBUILD_VIDEO_CLIP_TRUST_METADATA_FLAG \
|
| 279 |
+
--video-clip-action-exec-horizon=$ACTION_EXEC_HORIZON \
|
| 280 |
+
"$@"
|
pyproject.toml
ADDED
|
@@ -0,0 +1,85 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
[project]
|
| 2 |
+
name = "psi"
|
| 3 |
+
version = "0.0.0"
|
| 4 |
+
description = "Psi: towards humanoid foundation model"
|
| 5 |
+
readme = "README.md"
|
| 6 |
+
requires-python = "==3.10.*"
|
| 7 |
+
dependencies = [
|
| 8 |
+
"albumentations==1.4.18",
|
| 9 |
+
"datasets==3.6.0",
|
| 10 |
+
"dm-tree==0.1.8",
|
| 11 |
+
"einops==0.8.1",
|
| 12 |
+
"einx>=0.3.0",
|
| 13 |
+
"h5py==3.14.0",
|
| 14 |
+
"imageio==2.34.2",
|
| 15 |
+
"jsonlines>=4.0.0",
|
| 16 |
+
"numpy<2.0.0",
|
| 17 |
+
"numpydantic==1.6.7",
|
| 18 |
+
"omegaconf>=2.3.0",
|
| 19 |
+
"opencv-python",
|
| 20 |
+
"plotly==6.2.0",
|
| 21 |
+
"pydantic==2.10.6",
|
| 22 |
+
"pydantic-yaml>=1.6.0",
|
| 23 |
+
"tqdm==4.67.1",
|
| 24 |
+
"typer>=0.19.1",
|
| 25 |
+
"tyro==0.9.32",
|
| 26 |
+
"vuer[all]>=0.0.68",
|
| 27 |
+
"wandb>=0.20.0",
|
| 28 |
+
"simplejpeg>=1.9.0",
|
| 29 |
+
"dotenv",
|
| 30 |
+
"gdown>=5.2.0",
|
| 31 |
+
]
|
| 32 |
+
|
| 33 |
+
[build-system]
|
| 34 |
+
requires = ["setuptools>=61.0"]
|
| 35 |
+
build-backend = "setuptools.build_meta"
|
| 36 |
+
|
| 37 |
+
[tool.setuptools.package-data]
|
| 38 |
+
psi = ["assets/*"]
|
| 39 |
+
|
| 40 |
+
[tool.uv.sources]
|
| 41 |
+
psi = { path = ".", editable = true }
|
| 42 |
+
simple = { path = "third_party/SIMPLE", editable = true }
|
| 43 |
+
|
| 44 |
+
[tool.uv.workspace]
|
| 45 |
+
members = ["packages/*"]
|
| 46 |
+
|
| 47 |
+
[dependency-groups]
|
| 48 |
+
serve = [
|
| 49 |
+
"fastapi",
|
| 50 |
+
"uvicorn",
|
| 51 |
+
]
|
| 52 |
+
|
| 53 |
+
viz = [
|
| 54 |
+
"transforms3d>=0.4.2",
|
| 55 |
+
"vuer[all]>=0.0.68",
|
| 56 |
+
"viser>=1.0.15",
|
| 57 |
+
#"pin>=3.8.0",
|
| 58 |
+
#"numpy<2.0.0",
|
| 59 |
+
]
|
| 60 |
+
|
| 61 |
+
psi = [
|
| 62 |
+
"setuptools>=61",
|
| 63 |
+
"torch==2.7.0",
|
| 64 |
+
"torchvision==0.22.0",
|
| 65 |
+
"torchcodec==0.4.0",
|
| 66 |
+
"transformers==4.57.0",
|
| 67 |
+
"deepspeed==0.17.1",
|
| 68 |
+
"flash_attn==2.7.4.post1",
|
| 69 |
+
"triton==3.3.0",
|
| 70 |
+
"accelerate==1.7.0",
|
| 71 |
+
"peft==0.17.1",
|
| 72 |
+
"qwen-vl-utils==0.0.14",
|
| 73 |
+
"sentencepiece",
|
| 74 |
+
"lerobot @ git+https://github.com/songlin/lerobot.git@09929d8057b044b53aecaf5c6d7eb71f99e8beb9"
|
| 75 |
+
]
|
| 76 |
+
|
| 77 |
+
#simple = [
|
| 78 |
+
# "simple[full]",
|
| 79 |
+
#]
|
| 80 |
+
|
| 81 |
+
[tool.uv.extra-build-dependencies]
|
| 82 |
+
flash-attn = [{ requirement = "torch", match-runtime = true }]
|
| 83 |
+
|
| 84 |
+
[project.scripts]
|
| 85 |
+
serve_psi0 = "psi.deploy.psi0_serve_simple:main"
|
uv.lock
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|