Tinghong-Ye22 commited on
Commit
d8aa1ac
·
verified ·
1 Parent(s): 13db37b

Add files using upload-large-folder tool

Browse files
Files changed (50) hide show
  1. .env.sample +16 -0
  2. .gitignore +60 -0
  3. .gitmodules +3 -0
  4. LICENSE +17 -0
  5. Psi0-训练阶段-state-action总览.md +216 -0
  6. README.md +532 -0
  7. cache_dance_video_omni_embeds.sh +13 -0
  8. cache_motion_example_video_omni_embeds.sh +22 -0
  9. cache_video_omni_embeds.py +996 -0
  10. cache_video_omni_embeds.sh +275 -0
  11. cache_video_omni_embeds_dense_stride1_direct_full.sh +523 -0
  12. cache_video_omni_embeds_dense_stride1_full.sh +63 -0
  13. cache_video_omni_embeds_full.sh +31 -0
  14. dataset/seed/g1/csv/220714/jog_backward_start_001__A025.csv +0 -0
  15. dataset/seed/g1/csv/220714/jog_backward_start_003__A025.csv +0 -0
  16. dataset/seed/g1/csv/220714/walk_forward_stop_003__A025.csv +0 -0
  17. dataset/seed/g1/csv/230215/idle_crouch_right_loop_R_003__A194.csv +0 -0
  18. dataset/seed/g1/csv/230306/crouch_idle_004__A244.csv +0 -0
  19. dataset/seed/g1/csv/240327/jump_twice_R_001__A533.csv +0 -0
  20. dataset/seed/g1/csv/240327/jump_twice_R_002__A533.csv +0 -0
  21. dataset/seed/g1/csv/240327/jump_twice_R_003__A533.csv +0 -0
  22. dataset/seed/g1/csv/240327/touch_toes_standing_R_001__A533.csv +0 -0
  23. dataset/seed/metadata/seed_metadata_v004_100.csv +101 -0
  24. dataset/seed/metadata/seed_metadata_v004_100_move_name.jsonl +100 -0
  25. flake.lock +27 -0
  26. flake.nix +105 -0
  27. infer.sh +39 -0
  28. infer_prompt.sh +213 -0
  29. infer_prompt_distill.sh +104 -0
  30. infer_prompt_short.sh +157 -0
  31. infer_render.sh +64 -0
  32. infer_render_auto.sh +711 -0
  33. plan_0.md +374 -0
  34. posttrain-v1-psi0-all-7task-tune-vlm-direct-dense-actionhead-L.sh +313 -0
  35. posttrain-v1-psi0-all-7task-tune-vlm-direct-dense-actionhead-M.sh +313 -0
  36. posttrain-v1-psi0-all-7task-tune-vlm-direct-dense-actionhead-perfect-resume.sh +259 -0
  37. posttrain-v1-psi0-all-7task-tune-vlm-direct-dense-full.sh +421 -0
  38. posttrain-v1-psi0-all-7task-tune-vlm-direct-dense.sh +305 -0
  39. posttrain-v1-psi0-all-text-tune-vlm.sh +282 -0
  40. posttrain-v1-psi0-all-tune-vlm-direct-dense.sh +297 -0
  41. posttrain-v1-psi0-all-tune-vlm.sh +282 -0
  42. posttrain-v1-psi0-all.sh +281 -0
  43. posttrain-v1-psi0-all_resume.sh +285 -0
  44. posttrain-v1-psi0-audio-clip-tune-vlm.sh +282 -0
  45. posttrain-v1-psi0-motion-distill.sh +349 -0
  46. posttrain-v1-psi0-text-ckpt-continue-video-clip-tune-vlm-direct-dense.sh +305 -0
  47. posttrain-v1-psi0-text-ckpt-continue-video-clip-tune-vlm.sh +290 -0
  48. prebuild-multimodal-records-cache.sh +280 -0
  49. pyproject.toml +85 -0
  50. uv.lock +0 -0
.env.sample ADDED
@@ -0,0 +1,16 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ HF_TOKEN=
2
+ WANDB_API_KEY=
3
+ WANDB_ENTITY=
4
+ PSI_HOME=/hfm
5
+ DATA_HOME=/hfm/data
6
+ HF_HOME=/hfm/cache
7
+ TORCH_HOME=/hfm/cache
8
+ UV_CACHE_DIR=/hfm/cache
9
+ HF_LEROBOT_HOME=/hfm/data/lerobot
10
+ OMP_NUM_THREADS=8
11
+ TOKENIZERS_PARALLELISM=false
12
+ DEEPSPEED_LOG_LEVEL=warning
13
+ CUDA_LAUNCH_BLOCKING=true
14
+ TF_CPP_MIN_LOG_LEVEL=3
15
+ AV_LOG_LEVEL=quiet
16
+ PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION=python
.gitignore ADDED
@@ -0,0 +1,60 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Python-generated files
2
+ __pycache__/
3
+ *.py[oc]
4
+ build/
5
+ dist/
6
+ wheels/
7
+ *.egg-info
8
+
9
+ # Virtual environments
10
+ .venv*
11
+
12
+ /data/
13
+ cache
14
+ *.mp4
15
+ ml-egodex_dataset/ # soft link to the egodex dataset
16
+ wandb/
17
+ runs/
18
+ checkpoints/
19
+ log/
20
+ playgrounds/
21
+ TODO.md
22
+ *.zip
23
+
24
+ runs_old/
25
+ tmp_dir/
26
+ .runs*
27
+
28
+
29
+ .env
30
+ .env.local
31
+
32
+ _lerobot_build
33
+ .cache
34
+ .data
35
+ .logs
36
+
37
+ *.local.yaml
38
+
39
+
40
+ vq/
41
+ extern/vq_bet_official/
42
+ src/fast/egodex-*
43
+ *.pkl
44
+ !src/fast/egodex-rel-50w-1x48-v2048-s100
45
+
46
+ .deepspeed_env
47
+ *.bak
48
+ #uv.lock
49
+ .vscode/
50
+ .direnv/
51
+ third_party/
52
+ src/gr00t/.venv/
53
+ props/
54
+ textures/
55
+ test.py
56
+ *.npy
57
+
58
+ unitree_sdk2_python
59
+
60
+ dataset/
.gitmodules ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ [submodule "third_party/SIMPLE"]
2
+ path = third_party/SIMPLE
3
+ url = git@github.com:songlin/SIMPLE.git
LICENSE ADDED
@@ -0,0 +1,17 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ Apache License
2
+ Version 2.0, January 2004
3
+ http://www.apache.org/licenses/
4
+
5
+ Copyright 2026 PSI-lab, USC
6
+
7
+ Licensed under the Apache License, Version 2.0 (the "License");
8
+ you may not use this file except in compliance with the License.
9
+ You may obtain a copy of the License at
10
+
11
+ http://www.apache.org/licenses/LICENSE-2.0
12
+
13
+ Unless required by applicable law or agreed to in writing, software
14
+ distributed under the License is distributed on an "AS IS" BASIS,
15
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16
+ See the License for the specific language governing permissions and
17
+ limitations under the License.
Psi0-训练阶段-state-action总览.md ADDED
@@ -0,0 +1,216 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ ---
2
+ tags:
3
+ - psi0
4
+ - training
5
+ - state-action
6
+ ---
7
+
8
+ # Psi0 训练阶段 state/action 总览
9
+
10
+ > [!summary]
11
+ > 当前开源代码里,`pretrain`、`posttrain`、`finetune-real`、`finetune-simple` 都不是“显式全身关节输入 + 显式全身关节输出”。
12
+ > 更接近的实际接口是:
13
+ > - 上肢:手 + 手臂
14
+ > - 躯干:`rpy` + `height`
15
+ > - 移动相关:`torso_vx` / `torso_vy` / `torso_vyaw` / `target_yaw`
16
+ > - 下肢:更多像交给低层 controller,而不是由这层模型直接输出腿关节轨迹
17
+
18
+ ## 1. 一页结论
19
+
20
+ | 阶段 | 模型输入 `state` | 模型输出 `action` | 是否显式包含腿关节输出 | 依据 |
21
+ | --- | --- | --- | --- | --- |
22
+ | `pretrain` `EgoDex` | `48` 维手部/腕部表示 | `48` 维手部/腕部表示 | 否 | `src/psi/data/egodex/egodex_dataset.py:308-330, 519-523` |
23
+ | `pretrain` `HE` | `hand_joints + arm_joints` | 左右腕位姿增量 + 双手指尖轨迹 | 否 | `src/psi/config/transform.py:112-212, 221-230` |
24
+ | `posttrain` `HE` | `hand_joints + arm_joints` | `action.joint_angles`,但当前接口仍是上肢维度 | 否 | `src/psi/data/humanoid/he_raw_dataset.py:206-229`;`src/psi/config/transform.py:718-755` |
25
+ | `finetune-real` | `hand + arm + torso_rpy + torso_height` | `hand + arm + rpy + height + vx + vy + vyaw + target_yaw` | 否 | `scripts/data/raw_to_lerobot.py:203-212, 239-258` |
26
+ | `finetune-simple` | `32` 维 psi-style `states`,本质是 `hand + arm + rpy + height` | `36` 维动作,核心仍是上肢 + 躯干/移动命令 | 否 | `src/gr00t/gr00t/deploy/gr00t_serve_simple.py:193-219, 133-166`;`src/psi/config/transform.py:376-393` |
27
+
28
+ ## 2. 各阶段拆开看
29
+
30
+ ### 2.1 `pretrain` on `EgoDex`
31
+
32
+ - `states` 是当前动作表示,不是全身关节。
33
+ - `actions` 是未来动作 chunk,也不是全身关节。
34
+ - 代码里直接写的是 `48d` action representation。
35
+
36
+ 维度含义:
37
+ - 左腕
38
+ - 左手指尖
39
+ - 右腕
40
+ - 右手指尖
41
+
42
+ 依据:
43
+ - `src/psi/data/egodex/egodex_dataset.py:308-330`
44
+ - `src/psi/data/egodex/egodex_dataset.py:519-523`
45
+
46
+ 结论:
47
+ - 这是手部/腕部操作表示,不是全身关节表示。
48
+
49
+ ### 2.2 `pretrain` on `HE`
50
+
51
+ - `state` 由 `observation.hand_joints` 和 `observation.arm_joints` 拼起来。
52
+ - `action` 由左右腕位姿和双手指尖轨迹拼起来。
53
+ - 没有把腿关节放进这条训练链路。
54
+
55
+ 依据:
56
+ - `src/psi/config/transform.py:221-230`
57
+ - `src/psi/config/transform.py:112-212`
58
+
59
+ 结论:
60
+ - 这条线仍然是上肢操作表示,不是全身关节表示。
61
+
62
+ ### 2.3 `posttrain` on `HE`
63
+
64
+ - 原始 HE dataset loader 只读:
65
+ - `hand_state`
66
+ - `arm_state`
67
+ - 没有在这里读 `leg_state` 进入训练样本。
68
+ - `HEPosttrainRepackTransform` 里:
69
+ - `states = hand_joints + arm_joints`
70
+ - `actions = action.joint_angles`
71
+ - 但当前接口和脚本配置仍然是上肢维度,不是完整腿部关节轨迹接口。
72
+
73
+ 依据:
74
+ - `src/psi/data/humanoid/he_raw_dataset.py:206-229`
75
+ - `src/psi/config/transform.py:718-755`
76
+
77
+ 结论:
78
+ - `posttrain` 也不是显式全身关节训练。
79
+
80
+ ### 2.4 `finetune-real`
81
+
82
+ - 数据转换脚本确实读了 `leg_state`,也把 `sol_q` 拆出了 `leg_joints`。
83
+ - 但最后真正写进 `states` / `action` 的内容里,没有把腿关节放进去。
84
+
85
+ `states` 实际组成:
86
+ - `hand_joints`
87
+ - `arm_joints`
88
+ - `torso_rpy`
89
+ - `torso_height`
90
+
91
+ `action` 实际组成:
92
+ - `hand_joints`
93
+ - `arm_joints`
94
+ - `rpy`
95
+ - `height`
96
+ - `torso_vx`
97
+ - `torso_vy`
98
+ - `torso_vyaw`
99
+ - `target_yaw`
100
+
101
+ 依据:
102
+ - `scripts/data/raw_to_lerobot.py:203-212`
103
+ - `scripts/data/raw_to_lerobot.py:239-258`
104
+
105
+ 训练器对 `36` 维动作的切分也和上面一致:
106
+ - `0:14` 手
107
+ - `14:28` 臂
108
+ - `28:31` `rpy`
109
+ - `31:32` `height`
110
+ - `32:33` `vx`
111
+ - `33:34` `vy`
112
+ - `34:35` `vyaw`
113
+ - `35:36` `target_yaw`
114
+
115
+ 依据:
116
+ - `src/psi/trainers/finetune.py:492-520`
117
+
118
+ 脚本配置也写死了 `36` 维:
119
+ - `scripts/train/psi0/finetune-real-psi0.sh:50-72`
120
+
121
+ 补充:
122
+ - 可视化脚本也把 real 数据的腿关节直接置零,不从预测动作里取。
123
+
124
+ 依据:
125
+ - `scripts/viz/viz_episode_real.py:73-75`
126
+ - `scripts/viz/viz_episode_real.py:90-94`
127
+
128
+ 结论:
129
+ - `finetune-real` 不是“全身关节预测”。
130
+ - 更准确地说,是“上肢关节 + 躯干/移动命令预测”。
131
+
132
+ ### 2.5 `finetune-simple`
133
+
134
+ - `simple` 原始观测里有腿关节,这点是有的。
135
+ - 但给 psi 模型用的 `states` 不是原始全身 `proprio`,而是压缩过的 psi-style 状态。
136
+
137
+ `states` 的 `32` 维组成:
138
+ - 左手 `7`
139
+ - 右手 `7`
140
+ - 左臂 `7`
141
+ - 右臂 `7`
142
+ - `rpy` `3`
143
+ - `height` `1`
144
+
145
+ 依据:
146
+ - `src/gr00t/gr00t/deploy/gr00t_serve_simple.py:193-219`
147
+
148
+ `action` 在 SIMPLE 服务侧被拆成:
149
+ - `left_hand`
150
+ - `right_hand`
151
+ - `left_arm`
152
+ - `right_arm`
153
+ - `rpy`
154
+ - `height`
155
+ - `torso_vx`
156
+ - `torso_vy`
157
+ - `torso_vyaw`
158
+ - `target_yaw`
159
+
160
+ 依据:
161
+ - `src/gr00t/gr00t/deploy/gr00t_serve_simple.py:133-166`
162
+
163
+ psi 的 `SimpleRepackTransform` 也只是把 `states` / `action` 取进来,再 pad 到训练���度:
164
+ - `src/psi/config/transform.py:376-393`
165
+
166
+ 训练脚本同样固定成 `36` 维:
167
+ - `scripts/train/psi0/finetune-simple-psi0.sh:50-72`
168
+
169
+ 可视化脚本里:
170
+ - `proprio` 会显示腿关节
171
+ - 但 `action` 里的腿是直接用当前观测腿关节,而不是用模型预测结果
172
+
173
+ 依据:
174
+ - `scripts/viz/viz_episode_simple.py:83-85`
175
+ - `scripts/viz/viz_episode_simple.py:93-96`
176
+
177
+ 结论:
178
+ - `finetune-simple` 这条线也不是“模型直接输出腿关节轨迹”。
179
+
180
+ ## 3. 下肢到底在哪里
181
+
182
+ > [!note]
183
+ > README 的说法和代码里的实际接口,要分开理解。
184
+
185
+ README 里写的是:
186
+ - `System-1` 预测 future whole-body action chunks
187
+ - `System-0` 执行 predicted lower-body action commands
188
+
189
+ 依据:
190
+ - `README.md:26`
191
+
192
+ 结合实际代码,更合理的理解是:
193
+ - `System-1` 负责高层动作生成
194
+ - 当前开源训练接口里,显式暴露给模型训练的主要是上肢和躯干/移动命令
195
+ - 下肢稳定控制更多由 `System-0` 的 tracking controller 负责
196
+
197
+ 所以如果问题是:
198
+
199
+ **“这份代码里有没有直接训练/输出全身每个关节?”**
200
+
201
+ 答案是:
202
+ - 没看到。
203
+
204
+ **“有没有和移动、下肢相关的信息?”**
205
+
206
+ 答案是:
207
+ - 有,但更像高层 locomotion command,不是显式腿关节序列。
208
+
209
+ ## 4. 最短版记忆卡
210
+
211
+ - `pretrain(EgoDex)`:手/腕 `48D`
212
+ - `pretrain(HE)`:`hand + arm`,动作是腕位姿 + 指尖轨迹
213
+ - `posttrain(HE)`:`hand + arm`
214
+ - `finetune-real`:`hand + arm + rpy + height + vx/vy/vyaw/target_yaw`
215
+ - `finetune-simple`:原始观测有腿,但 psi 模型接口仍不是显式腿关节输出
216
+ - 结论:当前开源接口不是“全身关节端到端直接预测”
README.md ADDED
@@ -0,0 +1,532 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ <h1 align="center">Ψ₀: An Open Foundation Model <br/> Towards Universal Humanoid Loco-Manipulation
2
+ </h1>
3
+
4
+ <p align="center">
5
+ <img src="assets/media/teaser.jpg" alt="Psi0 teaser image" />
6
+ </p>
7
+
8
+ <div align="center">
9
+
10
+ [![arXiv](https://img.shields.io/badge/arXiv-2603.12263-df2a2a.svg)](https://arxiv.org/abs/2603.12263)
11
+ [![Static Badge](https://img.shields.io/badge/Project-Page-a)](https://psi-lab.ai/Psi0)
12
+ [![Model](https://img.shields.io/badge/Hugging%20Face-Model-yellow)](https://huggingface.co/USC-PSI-Lab/psi-model)
13
+ [![Data](https://img.shields.io/badge/Hugging%20Face-Data-pink)](https://huggingface.co/datasets/USC-PSI-Lab/psi-data)
14
+ [![License](https://img.shields.io/badge/License-Apache2.0-blue.svg)](./LICENSE)
15
+
16
+ </div>
17
+
18
+ Contributors: [Songlin Wei](https://songlin.github.io/), [Hongyi Jing](https://hongyijing.me/), [Boqian Li](https://boqian-li.github.io/), [Zhenyu Zhao](https://zhenyuzhao.com/), [Jiageng Mao](https://pointscoder.github.io/), [Zhenhao Ni](https://nizhenhao-3.github.io/) , [Sicheng He](https://hesicheng.net/), [Jie Liu](https://jie0530.github.io/), [Xiawei Liu](https://www.xiaweiliu.com/), Kaidi Kang, Sheng Zang,[Weiduo Yuan](https://weiduoyuan.com/), [Marco Pavone](https://profiles.stanford.edu/marco-pavone), Di Huang, [Yue Wang](https://yuewang.xyz/)
19
+
20
+ -------
21
+
22
+ $\Psi_0$ is an open vision-language-action (VLA) model for dexterous humanoid loco-manipulation. Our model first learns task semantics and visual representation from large-scale human egocentic videos, and then is post-trained on a smaller amount of real-world teleoperated robot data, to learn general dynamics of the embodiment.
23
+
24
+ Our foundation model is capable of acquiring new long-horizontal dexterous loco-manipulation skill by fine-tuning using as few as 80 trajectories. ***Our key finding is that scaling the right data in the right way.***
25
+
26
+ At the top, the $\Psi_0$ model consists of two end-to-end trained components: a vision–language backbone (System-2) and a multimodal diffusion transformer (System-1) action expert. The backbone is based on Qwen’s Qwen3-VL-2B-Instruct, which extracts vision–language features from observations and instructions. These features condition a flow-based multimodal diffusion transformer inspired by Stable Diffusion 3. The action expert (≈500M parameters) predicts future whole-body action chunks, enabling efficient fusion of visual, linguistic, and action representations. At the lowest level (System-0), an RL-based tracking controller executes the predicted lower-body action commands, ensuring stable and precise physical control.
27
+
28
+ <p align="center">
29
+ <img src="assets/media/arch.png" alt="Psi0 model" />
30
+ </p>
31
+
32
+ ## Table of Contents
33
+ <!-- - [Installation](#-environment-setup) -->
34
+ <!-- - [Pre- & Post- Training](#-) -->
35
+ <!-- - [Data Pre-Processing](#-) -->
36
+ - [Finetune Ψ₀ on Unitree G1 Humanoid Robot](#finetune-psi0)
37
+ - [Installation](#installation)
38
+ - [Data Collection](#data-collection)
39
+ - [Fine-Tuning](#training-real)
40
+ - [Open-Loop Evaluation](#open-loop-evaluation)
41
+ - [Deployment](#deployment)
42
+ - [Baselines](#baselines)
43
+ - [GR00T N1.6](#groot-n16)
44
+ - [OpenPi π0.5](#openpi-05)
45
+ - [InternVLA-M1](#internvla-m1)
46
+ - [H-RDT](#-)
47
+ - [EgoVLA](#-)
48
+ - [Diffusion Policy](#-)
49
+ - [ACT](#-)
50
+ - [Simulation](#simulation)
51
+ - [Install SIMPLE](#install-simple)
52
+ - [Data Generation](#data-generation)
53
+ - [Fine-Tuning](#training-sim)
54
+ - [Evaluation in SIMPLE](#evaluation-in-simple)
55
+ - [Reproduce Ψ₀: Pre-Training and Post-Training](#pre-post-train)
56
+ - [Checkpoints](#checkpoints)
57
+ - [Troubleshootings](#troubleshootings)
58
+ - [Citation](#️-citation)
59
+
60
+ <a id="finetune-psi0"></a>
61
+ ## Finetune Ψ₀ on Unitree G1 Humanoid Robot
62
+
63
+ ### Installation
64
+
65
+ Clone the project and change directory to the project root:
66
+ ```bash
67
+ git clone git@github.com:physical-superintelligence-lab/Psi0.git
68
+ cd Psi0
69
+ ```
70
+ We use [uv](https://docs.astral.sh/uv/getting-started/installation/) to manage Python dependencies. Install `uv` if not already installed:
71
+
72
+ ```bash
73
+ curl -LsSf https://astral.sh/uv/install.sh | sh
74
+ ```
75
+
76
+ Set up the $\Psi_0$ environment:
77
+
78
+ > ℹ️ We manage the $\Psi_0$ environment and all the baselines through `uv` and they all share the same `src/` code. See [Environment Management](baselines/README.md) for more details.
79
+
80
+ ```
81
+ uv venv .venv-psi --python 3.10
82
+ source .venv-psi/bin/activate
83
+ GIT_LFS_SKIP_SMUDGE=1 uv sync --all-groups --index-strategy unsafe-best-match --active
84
+ uv pip install flash_attn==2.7.4.post1 --no-build-isolation
85
+ ```
86
+
87
+ Test installation, a version number should be displayed.
88
+ ```bash
89
+ python -c "import psi;print(psi.__version__)"
90
+ ```
91
+
92
+ Verify the shared `lerobot` stack is importable.
93
+ ```bash
94
+ python -c "from psi.data.lerobot.compat import LEROBOT_LAYOUT; print(LEROBOT_LAYOUT)"
95
+ ```
96
+
97
+ ### Data Collection
98
+ > 📂 We open-sourced all the 9 real-world tasks. You can directly download the data and jump to the [Fine-Tuning](#training-real).
99
+
100
+ > 🔥 We first release our internal test data collection pipeline which uses Apple Vision Pro to teleoperate Unitree G1 humanoid Robot with two Dex3-1 hands.
101
+
102
+ See the detailed teleoperation guide here:
103
+ [Real-World Teleoperation Guide](real/README.md)
104
+
105
+
106
+ #### Pre-Processing: Convert Raw Data to LeRobot Format
107
+
108
+ ```
109
+ export task=Hug_box_and_move
110
+
111
+ hf download USC-PSI-Lab/psi-data \
112
+ g1_real_raw/$task.zip \
113
+ --local-dir=$PSI_HOME/data/real_teleop_g1 \
114
+ --repo-type=dataset
115
+
116
+ unzip $PSI_HOME/data/real_teleop_g1/g1_real_raw/$task.zip -d $PSI_HOME/data/real_teleop_g1/g1_real_raw/$task
117
+ ```
118
+ You should observe similar folder structure:
119
+
120
+ ```
121
+ g1_real_raw
122
+ └── Hug_box_and_move
123
+ ├── episode_0
124
+ │   ├── color
125
+ │   │   ├── frame_000000.jpg
126
+ │   │   └── ...
127
+ │   └── data.json
128
+ └── ...
129
+ ```
130
+
131
+ Edit the task description file with the following format, eg.,
132
+ ```
133
+ vim scripts/data/task_description_dict.json
134
+ ```
135
+ ```
136
+ {
137
+ "Hug_box_and_move": "Hug box and move."
138
+ }
139
+ ```
140
+
141
+ Run conversion script
142
+ ```
143
+ python scripts/data/raw_to_lerobot.py \
144
+ --data-root=/hfm/data/real_teleop_g1/g1_real_raw \
145
+ --work-dir=/hfm/data/real \
146
+ --repo-id=psi0-real-g1 \
147
+ --robot-type=g1 \
148
+ --task=$task
149
+ ```
150
+
151
+ Calculate stats
152
+ ```
153
+ python scripts/data/calc_modality_stats.py \
154
+ --work-dir=$PSI_HOME/data/real \
155
+ --task=$task
156
+ ```
157
+
158
+ Create **$\Psi_0$** format stats (simply a copy for now)
159
+ ```
160
+ cp $PSI_HOME/data/real/$task/meta/stats.json $PSI_HOME/data/real/$task/meta/stats_psi0.json
161
+ ```
162
+
163
+ Now it's ready to finetune $\Psi_0$.
164
+
165
+ > ✈️ If training env is already configured, directly launch training via `scripts/train/psi0/finetune-real-psi0.sh $task`
166
+
167
+
168
+ <a id="training-real"></a>
169
+ ### Fine-Tuning
170
+
171
+ > ✔️ Suppose the data is already collected and processed. Now we can proceed to fine-tune the $\Psi_0$ model.
172
+
173
+ > There is a [known issue](https://github.com/physical-superintelligence-lab/Psi0/issues/3) of loading our real data, apply this fix first `python scripts/data/patch_lerobot_meta.py $PSI_HOME/data/real/$task`
174
+
175
+ > 📝 Here we illustrate by using the pre-collected data from [Huggingface psi-data](https://huggingface.co/datasets/USC-PSI-Lab/psi-data/tree/main/real).
176
+
177
+ Set up the environment variables following `.env.sample`. The environment variables will be loaded by the `dotenv.load_dotenv()` in python.
178
+ ```
179
+ cp .env.sample .env
180
+ # and edit the following env variables
181
+ # HF_TOKEN=<YOUR HF READ TOKEN>
182
+ # WANDB_API_KEY=<API KEY for wandb logging>
183
+ # WANDB_ENTITY=<wandb entity>
184
+ # PSI_HOME=<Path where PSI cache/checkpoint/data are located by convention>
185
+
186
+ source .env
187
+ echo $PSI_HOME
188
+ ```
189
+
190
+ Download the collected real-world data and extract it:
191
+ ```
192
+ export task=Pick_bottle_and_turn_and_pour_into_cup
193
+
194
+ hf download USC-PSI-Lab/psi-data \
195
+ real/$task.zip \
196
+ --local-dir=$PSI_HOME/data \
197
+ --repo-type=dataset
198
+
199
+ unzip $PSI_HOME/data/real/$task.zip -d $PSI_HOME/data/real
200
+ ```
201
+ > 👀 If you want to visualize the episode please refer to the [Data Visualization](examples/visualize.md) in the examples.
202
+
203
+ Launch the training script:
204
+ ```
205
+ scripts/train/psi0/finetune-real-psi0.sh $task
206
+ ```
207
+
208
+ > 🖥️ You can always change the GPUs, e.g., `CUDA_VISIBLE_DEVICES=0,1,2,3 scripts/train/...`.
209
+
210
+ > ⚠️ Please try to maintain a reasonable global batch size = device batch size x number of GPUs x gradient accumulation step. We use global batch size 128 throughout all the real-world and simulation experiments.
211
+
212
+
213
+ ### Open-Loop Evaluation
214
+ > Follow the steps in `examples/simple/openloop_eval.ipynb`
215
+
216
+ Load the training dataset, and run model inference to see how model fits the training data.
217
+
218
+ ### Deployment
219
+
220
+ #### Serve $\Psi_0$ (RTC mode)
221
+
222
+ ```bash
223
+ bash ./scripts/deploy/serve_psi0-rtc.sh
224
+ ```
225
+
226
+ #### Start $\Psi_0$ Client (RTC mode)
227
+
228
+ ```bash
229
+ bash ./real/scripts/deploy_psi0-rtc.sh
230
+ ```
231
+
232
+ For detailed real-world deployment environment setup, please also refer to the dedicated documentation:
233
+
234
+ [Real-World Teleoperation Guide](real/README.md)
235
+
236
+
237
+ ## Baselines
238
+
239
+ <a id="groot-n16"></a>
240
+
241
+ ### GR00T
242
+ Install the env
243
+ ```bash
244
+ cd src/gr00t; uv sync
245
+ ```
246
+ 1. training
247
+ ```bash
248
+ cd src/gr00t
249
+ ./scripts/train_gr00t.sh --dataset-path /your/lerobot/dataset
250
+ ```
251
+ 2. serving a checkpoint
252
+ ```bash
253
+ cd src/gr00t
254
+ ./scripts/deploy_gr00t.sh
255
+ ```
256
+
257
+ 3. openloop eval on trained checkpoint using gt
258
+ ```bash
259
+ cd src/gr00t
260
+ ./scripts/openloop_eval.sh
261
+ ```
262
+
263
+ <a id="openpi-05"></a>
264
+
265
+ ### OpenPI $\pi_{0.5}$
266
+
267
+ Please see more detailed instructions here: [baselines/pi05](baselines/pi05/README.md).
268
+
269
+ ### InternVLA-M1
270
+ Install the env
271
+ ```bash
272
+ cd src/InternVLA-M1; uv sync --python 3.10
273
+ ```
274
+ 1. training
275
+ ```bash
276
+ cd src/InternVLA-M1
277
+ bash scripts/train_internvla.sh
278
+ ```
279
+ 2. serving a checkpoint
280
+ ```bash
281
+ cd src/InternVLA-M1
282
+ ./scripts/deploy_internvla.sh
283
+ ```
284
+
285
+ ## Simulation
286
+
287
+ We use [SIMPLE]() to benchmark $\Psi_0$ and all the baselines.
288
+
289
+ > 📢 SIMPLE is an easy-to-use humanoid benchmarking simulator built on the MuJoCo physics engine and Isaac Sim rendering.
290
+
291
+ ### Install SIMPLE
292
+ [Coming soon]
293
+
294
+ ### Data Generation
295
+ > 📂 We also provide 5 pre-collected whole-body humanoid loco-manipulation tasks at [Huggingface psi-data](https://huggingface.co/datasets/USC-PSI-Lab/psi-data/tree/main/real). If you want to use the existing simulation data, jump to the [Fine-Tuning](#training-sim)
296
+
297
+ #### Motion-Planning Based Data Generation
298
+ [Coming soon]
299
+
300
+ #### Teleoperation in Simulator
301
+ [Coming soon]
302
+
303
+ <a id="training-sim"></a>
304
+ ### Fine-Tuning
305
+
306
+ Download [SIMPLE task data](https://huggingface.co/datasets/USC-PSI-Lab/psi-data/tree/main/simple) and extract it:
307
+
308
+ > 💡 Dont forget `source .env` first before following below commands.
309
+
310
+ ```
311
+ export task=G1WholebodyBendPick-v0
312
+
313
+ hf download USC-PSI-Lab/psi-data \
314
+ simple/$task$.zip \
315
+ --local-dir=$PSI_HOME/data \
316
+ --repo-type=dataset
317
+
318
+ unzip $PSI_HOME/data/simple/$task.zip -d $PSI_HOME/data/simple
319
+ ```
320
+
321
+ > 👀 If you want to visualize the episode please refer to the [Data Visualization](examples/visualize.md) in the examples.
322
+
323
+ Start training:
324
+
325
+ > Please [set up the envrionment variables](#training-real) if not done so yet.
326
+
327
+ ```
328
+ bash scripts/train/psi0/finetune-simple-psi0.sh $task
329
+ ```
330
+ The training will create a run dir which is located under `.runs` in the project root.
331
+ If your GPU has limited VRAM, set `--train.optimizer-foreach=false` to reduce optimizer-step memory usage at the cost of some speed.
332
+
333
+ ### Evaluation in SIMPLE
334
+
335
+ #### Serve $\Psi_0$
336
+ ```
337
+ export run_dir=<the run dir here under folder .runs>
338
+ export ckpt_step=<checkpoint step>
339
+ uv run --active --group psi --group serve serve_psi0 \
340
+ --host 0.0.0.0 \
341
+ --port 22085 \
342
+ --run-dir=$run_dir \
343
+ --ckpt-step=$ckpt_step \
344
+ --action-exec-horizon=24 \
345
+ --rtc
346
+ ```
347
+
348
+
349
+ Run open-loop evaluation (offline)
350
+
351
+ [examples/simple/openloop_eval.ipynb](examples/simple/openloop_eval.ipynb)
352
+
353
+ #### Run the Evaluation Client
354
+ > If the server is started on a remote server, run ssh port forward. eg., `ssh -L 22086:localhost:22086 songlin@nebula100`.
355
+
356
+ > Once port forward is done, open a new terminal to test if server is up `curl -i http://localhost:22085/health`
357
+
358
+ Launch the eval client through docker
359
+ ```
360
+ GPUs=1 docker compose run eval $task psi0 \
361
+ --host=localhost \
362
+ --port=22085 \
363
+ --sim-mode=mujoco_isaac \
364
+ --headless \
365
+ --max-episode-steps=360 \
366
+ --num-episodes=10 \
367
+ --data-format=lerobot \
368
+ --data-dir=data/$task
369
+ ```
370
+ The policy rollout videos will be found in folder `third_party/SIMPLE/data/evals/psi0`.
371
+
372
+ > The evaluation for a single episode could take up to 6~10 minutes because SIMPLE use a synchronous rendering API in IsaacSim. See here for [more explanation](#).
373
+
374
+ <a id="pre-post-train"></a>
375
+ ## Reproduce Ψ₀: Pre-Training and Post-Training
376
+
377
+
378
+ ### Pre-Train VLM
379
+
380
+ Download and cache the official `Qwen/Qwen3-VL-2B-Instruct` weights.
381
+ ```
382
+ scripts/predownload_qwen3vl.py
383
+ ```
384
+
385
+ Pre-train on the [EgoDex dataset](https://github.com/apple/ml-egodex)
386
+
387
+ Pre-compute `48 DoF EgoDex action`:
388
+
389
+ > We re-use the pre-process code from [H-RDT EgoDex Pre-Processing](https://github.com/HongzheBi/H_RDT?tab=readme-ov-file#data-preprocessing).
390
+ > 1. Change the paths in `src/h_rdt/datasets/pretrain/setup_pretrain.sh`.
391
+ > 2. Tweak the `NUM_PROCESSES` if on a powerful server, i tried max 64.
392
+ > 3. set `FORCE_OVERWRITE=True` if the processing script is disrupted.
393
+
394
+ ```
395
+ source src/h_rdt/datasets/pretrain/setup_pretrain.sh
396
+ source .venv-psi/bin/activate
397
+ bash src/h_rdt/datasets/pretrain/run_pretrain_pipeline.sh
398
+ ```
399
+
400
+ ```
401
+ bash scripts/train/psi0/pretrain-egodex-psi0-fast.sh
402
+ ```
403
+
404
+ Pre-train on [humanoid everyday dataset](https://huggingface.co/datasets/USC-GVL/humanoid-everyday)
405
+
406
+ > Please download the pre-processed HE data here: `hf download USC-PSI-Lab/psi-data HE_RAW.zip --repo-type=dataset`
407
+
408
+ ```
409
+ bash scripts/train/psi0/pretrain-he-psi0-fast.sh
410
+ ```
411
+
412
+ Save the pretrained checkpoints once training is done:
413
+ ```
414
+ python scripts/save_pretrain_qwen3vl_backbone.py
415
+ ```
416
+
417
+ ### Post-Train Action Expert
418
+
419
+ Download pre-trained `psi-0` VLM backbone
420
+ ```
421
+ python scripts/data/download.py \
422
+ --repo-id=USC-PSI-Lab/psi-model \
423
+ --remote-dir=pre.fast.egodex.2512241941/pretrained/ckpt_200000 \
424
+ --local-dir=/hfm/cache/checkpoints/psi0/pre.fast.egodex.2512241941.ckpt200k \
425
+ --repo-type=model
426
+ ```
427
+
428
+ Post-train on [humanoid everyday (HE) dataset](https://huggingface.co/datasets/USC-GVL/humanoid-everyday)
429
+ ```
430
+ bash scripts/train/psi0/posttrain-he-psi0.sh
431
+ ```
432
+
433
+ Save post-trained action header once training is over
434
+ ```
435
+ python scripts/save_posttrain_action_expert.py
436
+ ```
437
+
438
+ ## Checkpoints
439
+
440
+ The released checkpoints on [HuggingFace Psi-Model](https://huggingface.co/USC-PSI-Lab/psi-model) is listed
441
+
442
+ | Checkpoint | Description | Remote Directory |
443
+ |---|---|---|
444
+ | $\Psi_0$ VLM<br/>(Baseline) | Pre-trained VLM backbone (EgoDex 200K steps + HE 30K steps) | `psi0/pre.fast.1by1.2601091803.ckpt.ego200k.he30k` |
445
+ | $\Psi_0$ Action Expert<br/>(Baseline) | Post-trained Action Expert On HE | `psi0/postpre.1by1.pad36.2601131206.ckpt.he30k` |
446
+
447
+ and more variants for ablation studies:
448
+ | Checkpoint | Description | Remote Directory |
449
+ |---|---|---|
450
+ | $\Psi_0$ VLM<br/>(Ablation Study) | Pre-trained VLM backbone only on EgoDex 200K steps | `psi0/pre.fast.egodex.2512241941.ckpt200k` |
451
+ | $\Psi_0$ VLM<br/>(Ablation Study) | Pre-trained VLM backbone only on HE 48K steps | `psi0/pre.abl.only.he.2512311516.48k` |
452
+ | $\Psi_0$ VLM<br/>(Ablation Study) | Pre-trained VLM backbone only on 10% EgoDex | `psi0/pre.abl.ego.10per.2602021632.46k` |
453
+ | $\Psi_0$ Action Expert<br/>(Ablation Study) | Post-train on HE by picking pre-trained variant `psi0/pre.abl.only.he.2512311516.48k` | `psi0/postpre.abl.only.he.2602050012` |
454
+ | $\Psi_0$ Action Expert<br/>(Ablation Study) | Post-train on HE by picking pre-trained variant `psi0/pre.abl.ego.10per.2602021632.46k` | `psi0/postpre.abl.ego.10per.2602050006` |
455
+
456
+
457
+ Download the selected models
458
+
459
+ ```
460
+ hf download USC-PSI-Lab/psi-model \
461
+ --remote-dir=<remote dictory on huggingface repo>
462
+ --local-dir=$PSI_HOME/cache/checkpoints \
463
+ --repo-type=model
464
+ ```
465
+
466
+ ## Troubleshootings
467
+
468
+ 1. Lerobot dataset issues: `stack(): argument 'tensors' (position 1) must be tuple of Tensors, not Column`
469
+
470
+ This usually means the environment is still on the legacy PSI `lerobot` stack. Resync the PSI env so it uses the
471
+ same `lerobot` and `datasets` versions as SIMPLE, then verify the import layout:
472
+
473
+ ```bash
474
+ source .venv-psi/bin/activate
475
+ uv sync --group psi --active
476
+ python -c "from psi.data.lerobot.compat import LEROBOT_LAYOUT; print(LEROBOT_LAYOUT)"
477
+ ```
478
+
479
+ 2. Fail to install `evdev`, `src/evdev/input.c:10:10: fatal error: Python.h: No such file or directory`
480
+
481
+ ```
482
+ sudo apt update
483
+ sudo apt install -y python3-dev python3-venv build-essential \
484
+ linux-headers-$(uname -r)
485
+ ```
486
+
487
+ 3. RuntimeError: Could not load libtorchcodec. Likely causes ...
488
+ ```
489
+ sudo apt-get install ffmpeg
490
+ ```
491
+
492
+ 4. ImportError: cannot import name 'Deprecated' from 'wandb.proto.wandb_telemetry_pb2'
493
+
494
+ re-install `wandb`
495
+ ```
496
+ source .venv-pusht/bin/activate
497
+ uv pip uninstall wandb
498
+ uv pip install wandb==0.18.0
499
+ ```
500
+
501
+ 5. support `sm_120` on newer GPUs like `5090` or `RTX 6000`, UserWarning: Ignoring invalid value for boolean flag CUDA_LAUNCH_BLOCKING: truevalid values are 0 or 1.
502
+
503
+ update `torch` and `flash-attn`
504
+ ```
505
+ uv pip install torch==2.7.0 torchvision==0.22.0 torchaudio==2.7.0 --index-url https://download.pytorch.org/whl/cu128
506
+ uv pip install flash-attn --no-build-isolation
507
+ ```
508
+
509
+ 6. Failed to download and build `lerobot ... `, Use `git lfs logs last` to view the log.
510
+
511
+ ```
512
+ GIT_LFS_SKIP_SMUDGE=1 uv ...
513
+ ```
514
+ ## Citation
515
+
516
+ ```
517
+ @misc{wei2026psi0,
518
+ title={$\Psi_0$: An Open Foundation Model Towards Universal Humanoid Loco-Manipulation},
519
+ author={Songlin Wei and Hongyi Jing and Boqian Li and Zhenyu Zhao and Jiageng Mao and Zhenhao Ni and Sicheng He and Jie Liu and Xiawei Liu and Kaidi Kang and Sheng Zang and Weiduo Yuan and Marco Pavone and Di Huang and Yue Wang},
520
+ year={2026},
521
+ eprint={2603.12263},
522
+ archivePrefix={arXiv},
523
+ primaryClass={cs.RO},
524
+ url={https://arxiv.org/abs/2603.12263},
525
+ }
526
+ ```
527
+
528
+ ## License
529
+
530
+ This project is licensed under the Apache License 2.0.
531
+
532
+ See the [LICENSE](LICENSE) file for details.
cache_dance_video_omni_embeds.sh ADDED
@@ -0,0 +1,13 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/usr/bin/env bash
2
+ set -euo pipefail
3
+
4
+ cd /cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/Psi0-yth-v13
5
+
6
+ export BUILD_DANCE=1
7
+ export BUILD_MOTION_EXAMPLE=0
8
+ export CACHE_DANCE=1
9
+ export CACHE_MOTION_EXAMPLE=0
10
+ export OVERWRITE_CLIPS=${OVERWRITE_CLIPS:-1}
11
+ export OVERWRITE_EMBEDS=${OVERWRITE_EMBEDS:-1}
12
+
13
+ exec bash cache_video_omni_embeds.sh
cache_motion_example_video_omni_embeds.sh ADDED
@@ -0,0 +1,22 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/usr/bin/env bash
2
+ set -euo pipefail
3
+
4
+ cd /cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/Psi0-yth-v18
5
+
6
+ export PSI_VIDEO_CLIP_DATASET_ROOT=${PSI_VIDEO_CLIP_DATASET_ROOT:-/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/videogen/psi0_video_clip_50f_rand8}
7
+ export PSI_VIDEO_DEMO_ROOT=${PSI_VIDEO_DEMO_ROOT:-/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/videogen/motion_example_0506_demo}
8
+
9
+ export BUILD_CLIPS=1
10
+ export CACHE_EMBEDS=1
11
+ export BUILD_DANCE=0
12
+ export CACHE_DANCE=0
13
+ export BUILD_MOTION_EXAMPLE=1
14
+ export CACHE_MOTION_EXAMPLE=1
15
+
16
+ export BUILD_NUM_WORKERS=${BUILD_NUM_WORKERS:-160}
17
+ export EMBED_NUM_WORKERS=${EMBED_NUM_WORKERS:-160}
18
+ export OVERWRITE_CLIPS=${OVERWRITE_CLIPS:-1}
19
+ export OVERWRITE_EMBEDS=${OVERWRITE_EMBEDS:-1}
20
+ export OMP_NUM_THREADS=${OMP_NUM_THREADS:-1}
21
+
22
+ exec bash cache_video_omni_embeds.sh
cache_video_omni_embeds.py ADDED
@@ -0,0 +1,996 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from __future__ import annotations
2
+
3
+ import argparse
4
+ import hashlib
5
+ import json
6
+ import os
7
+ import socket
8
+ import time
9
+ import uuid
10
+ from concurrent.futures import FIRST_COMPLETED, ProcessPoolExecutor, ThreadPoolExecutor, as_completed, wait
11
+ from itertools import islice
12
+ from pathlib import Path
13
+
14
+ import numpy as np
15
+ from tqdm import tqdm
16
+ from transformers import Qwen2_5OmniProcessor
17
+
18
+ from psi.data.seed.seed_video_motion_dataset import (
19
+ load_video_array,
20
+ load_video_clip_array,
21
+ video_embed_cache_path,
22
+ )
23
+
24
+
25
+ REQUIRED_CACHE_KEYS = (
26
+ "pixel_values_videos",
27
+ "video_grid_thw",
28
+ "video_second_per_grid",
29
+ "video_sample_fps",
30
+ "video_size_pixels",
31
+ "source_video_fps",
32
+ "video_duration_sec",
33
+ )
34
+
35
+ _WORKER_PROCESSOR = None
36
+ _WORKER_MODEL_PATH = ""
37
+ _SOURCE_PATH_EXISTS: dict[str, bool] = {}
38
+
39
+
40
+ def parse_args() -> argparse.Namespace:
41
+ parser = argparse.ArgumentParser(description="Cache Qwen2.5-Omni video processor outputs.")
42
+ parser.add_argument("--video-root", type=str, default="")
43
+ parser.add_argument("--dataset-root", type=str, default="")
44
+ parser.add_argument("--metadata-jsonl", type=str, default="")
45
+ parser.add_argument("--video-path-key", type=str, default="video_path")
46
+ parser.add_argument("--embed-root", type=str, required=True)
47
+ parser.add_argument("--model-path", type=str, required=True)
48
+ parser.add_argument("--video-sample-fps", type=float, default=1.0)
49
+ parser.add_argument("--video-size-pixels", type=int, default=50176)
50
+ parser.add_argument("--video-decoder", type=str, default="ffmpeg", choices=["opencv", "ffmpeg"])
51
+ parser.add_argument("--dtype", type=str, default="float16", choices=["float16", "float32"])
52
+ parser.add_argument("--num-workers", type=int, default=1)
53
+ parser.add_argument("--save-compressed", action="store_true", default=False)
54
+ parser.add_argument(
55
+ "--existing-cache-check",
56
+ choices=["strict", "size"],
57
+ default=os.environ.get("PSI_EXISTING_CACHE_CHECK", "strict"),
58
+ help="How to validate existing npz caches before skipping. strict opens npz; size only checks final file size.",
59
+ )
60
+ parser.add_argument("--limit", type=int, default=0)
61
+ parser.add_argument("--overwrite", action="store_true", default=False)
62
+ parser.add_argument("--shard-num", type=int, default=1)
63
+ parser.add_argument("--shard-id", type=int, default=0)
64
+ parser.add_argument("--input-already-sharded", action="store_true", default=False)
65
+ parser.add_argument(
66
+ "--direct-source-clips",
67
+ action="store_true",
68
+ default=False,
69
+ help=(
70
+ "Read source_video_path + chunk_start_sec/duration from metadata, but write cache "
71
+ "to the canonical path derived from video_path."
72
+ ),
73
+ )
74
+ return parser.parse_args()
75
+
76
+
77
+ def _resolve_path(path_value: str, base_dir: Path) -> Path:
78
+ path = Path(str(path_value))
79
+ return path if path.is_absolute() else base_dir / path
80
+
81
+
82
+ def resolve_metadata_context(args: argparse.Namespace) -> tuple[Path, Path]:
83
+ dataset_root = Path(args.dataset_root) if args.dataset_root else Path(args.metadata_jsonl).parent
84
+ metadata_path = Path(args.metadata_jsonl)
85
+ if not metadata_path.is_absolute():
86
+ metadata_path = dataset_root / metadata_path
87
+ if not metadata_path.exists():
88
+ raise FileNotFoundError(f"metadata_jsonl does not exist: {metadata_path}")
89
+ return dataset_root, metadata_path
90
+
91
+
92
+ def build_video_task_from_row(
93
+ row: dict,
94
+ args: argparse.Namespace,
95
+ dataset_root: Path,
96
+ metadata_path: Path,
97
+ line_no: int,
98
+ ) -> dict:
99
+ video_value = row.get(args.video_path_key) or row.get("video_path") or row.get("original_video")
100
+ if not video_value:
101
+ raise ValueError(f"Missing video path at {metadata_path}:{line_no}")
102
+ video_path = _resolve_path(str(video_value), dataset_root)
103
+
104
+ task = {"video_path": video_path}
105
+ if args.direct_source_clips:
106
+ source_value = row.get("source_video_path")
107
+ if not source_value:
108
+ raise ValueError(f"Missing source_video_path at {metadata_path}:{line_no}")
109
+ source_video_path = _resolve_path(str(source_value), dataset_root)
110
+ source_key = str(source_video_path)
111
+ exists = _SOURCE_PATH_EXISTS.get(source_key)
112
+ if exists is None:
113
+ exists = source_video_path.exists()
114
+ _SOURCE_PATH_EXISTS[source_key] = exists
115
+ if not exists:
116
+ raise FileNotFoundError(f"Source video file does not exist: {source_video_path}")
117
+ task.update(
118
+ {
119
+ "decode_video_path": source_video_path,
120
+ "clip_start_sec": float(row.get("chunk_start_sec") or 0.0),
121
+ "clip_duration_sec": float(row.get("duration") or row.get("video_duration_sec") or 0.0),
122
+ }
123
+ )
124
+ else:
125
+ if not video_path.exists():
126
+ raise FileNotFoundError(f"Video file does not exist: {video_path}")
127
+ task["decode_video_path"] = video_path
128
+ return task
129
+
130
+
131
+ def iter_metadata_tasks_stream(
132
+ args: argparse.Namespace,
133
+ start_index: int = 0,
134
+ ):
135
+ dataset_root, metadata_path = resolve_metadata_context(args)
136
+ stop_index = int(args.limit) if args.limit > 0 else None
137
+ with metadata_path.open("r", encoding="utf-8") as f:
138
+ iterator = islice(f, start_index, None) if start_index > 0 else f
139
+ for task_index, line in enumerate(iterator, start=start_index):
140
+ if stop_index is not None and task_index >= stop_index:
141
+ break
142
+ line = line.strip()
143
+ if not line:
144
+ continue
145
+ row = json.loads(line)
146
+ task = build_video_task_from_row(
147
+ row=row,
148
+ args=args,
149
+ dataset_root=dataset_root,
150
+ metadata_path=metadata_path,
151
+ line_no=task_index + 1,
152
+ )
153
+ task["task_index"] = task_index
154
+ yield task
155
+
156
+
157
+ def iter_video_tasks(args: argparse.Namespace) -> list[dict]:
158
+ if args.metadata_jsonl:
159
+ dataset_root, metadata_path = resolve_metadata_context(args)
160
+
161
+ tasks = []
162
+ seen = set()
163
+ with metadata_path.open("r", encoding="utf-8") as f:
164
+ for line_no, line in enumerate(f, start=1):
165
+ line = line.strip()
166
+ if not line:
167
+ continue
168
+ row = json.loads(line)
169
+ task = build_video_task_from_row(
170
+ row=row,
171
+ args=args,
172
+ dataset_root=dataset_root,
173
+ metadata_path=metadata_path,
174
+ line_no=line_no,
175
+ )
176
+ video_path = task["video_path"]
177
+
178
+ key = str(video_embed_cache_path(
179
+ video_path,
180
+ args.embed_root,
181
+ video_sample_fps=args.video_sample_fps,
182
+ video_size_pixels=args.video_size_pixels,
183
+ ))
184
+ if key not in seen:
185
+ seen.add(key)
186
+ tasks.append(task)
187
+ return tasks
188
+
189
+ if not args.video_root:
190
+ raise ValueError("Either --metadata-jsonl or --video-root is required")
191
+ video_root = Path(args.video_root)
192
+ if not video_root.exists():
193
+ raise FileNotFoundError(f"video_root does not exist: {video_root}")
194
+ return [{"video_path": path, "decode_video_path": path} for path in sorted(video_root.glob("*.mp4"))]
195
+
196
+
197
+ def is_valid_cache(path: Path) -> bool:
198
+ if not path.exists() or path.stat().st_size <= 0:
199
+ return False
200
+ try:
201
+ with np.load(path, allow_pickle=False) as cache:
202
+ for key in REQUIRED_CACHE_KEYS:
203
+ if key not in cache:
204
+ return False
205
+ pixel_values = cache["pixel_values_videos"]
206
+ grid = cache["video_grid_thw"]
207
+ if pixel_values.size <= 0 or grid.size <= 0:
208
+ return False
209
+ except Exception:
210
+ return False
211
+ return True
212
+
213
+
214
+ def is_existing_cache_ready(path: Path, check_mode: str) -> bool:
215
+ if not path.exists() or path.stat().st_size <= 0:
216
+ return False
217
+ if check_mode == "size":
218
+ return True
219
+ return is_valid_cache(path)
220
+
221
+
222
+ def is_existing_cache_ready_fast(path: Path, check_mode: str) -> bool:
223
+ if check_mode == "size":
224
+ try:
225
+ return path.stat().st_size > 0
226
+ except FileNotFoundError:
227
+ return False
228
+ return is_existing_cache_ready(path, check_mode)
229
+
230
+
231
+ def remove_path(path: Path) -> None:
232
+ try:
233
+ path.unlink()
234
+ except FileNotFoundError:
235
+ pass
236
+
237
+
238
+ def stable_hash_mod(value: str, modulo: int) -> int:
239
+ digest = hashlib.sha1(value.encode("utf-8")).digest()
240
+ return int.from_bytes(digest[:8], "big") % int(modulo)
241
+
242
+
243
+ def shard_manifest_path(embed_root: Path, shard_id: int, shard_num: int) -> Path:
244
+ return embed_root / f"manifest.shard-{shard_id:05d}-of-{shard_num:05d}.json"
245
+
246
+
247
+ def resume_state_path(embed_root: Path, shard_id: int, shard_num: int) -> Path:
248
+ if shard_num > 1:
249
+ return embed_root / f"resume.shard-{shard_id:05d}-of-{shard_num:05d}.json"
250
+ return embed_root / "resume.json"
251
+
252
+
253
+ def write_json_atomic(path: Path, payload: dict) -> None:
254
+ tmp_path = path.with_name(f".{path.name}.{os.getpid()}.{uuid.uuid4().hex}.tmp")
255
+ tmp_path.write_text(
256
+ json.dumps(payload, ensure_ascii=False, indent=2),
257
+ encoding="utf-8",
258
+ )
259
+ tmp_path.replace(path)
260
+
261
+
262
+ def merge_shard_manifests(embed_root: Path, shard_num: int) -> bool:
263
+ shard_paths = [shard_manifest_path(embed_root, shard_id, shard_num) for shard_id in range(shard_num)]
264
+ if any(not path.exists() for path in shard_paths):
265
+ return False
266
+
267
+ manifests = []
268
+ for path in shard_paths:
269
+ try:
270
+ manifests.append(json.loads(path.read_text(encoding="utf-8")))
271
+ except Exception:
272
+ return False
273
+ if not manifests:
274
+ return False
275
+
276
+ merged = dict(manifests[0])
277
+ merged["count"] = int(sum(int(item.get("count") or 0) for item in manifests))
278
+ if all(bool(item.get("input_already_sharded")) for item in manifests):
279
+ merged["total_count"] = int(sum(int(item.get("count") or 0) for item in manifests))
280
+ else:
281
+ merged["total_count"] = int(max(int(item.get("total_count") or 0) for item in manifests))
282
+ merged["shard_id"] = None
283
+ merged["shard_num"] = int(shard_num)
284
+ merged["shards"] = [path.name for path in shard_paths]
285
+ write_json_atomic(embed_root / "manifest.json", merged)
286
+ return True
287
+
288
+
289
+ def maybe_load_json(path: Path) -> dict | None:
290
+ if not path.exists():
291
+ return None
292
+ try:
293
+ return json.loads(path.read_text(encoding="utf-8"))
294
+ except Exception:
295
+ return None
296
+
297
+
298
+ def load_known_total_count(args: argparse.Namespace, embed_root: Path) -> int | None:
299
+ manifest_path = (
300
+ shard_manifest_path(embed_root, args.shard_id, args.shard_num)
301
+ if args.shard_num > 1
302
+ else embed_root / "manifest.json"
303
+ )
304
+ payload = maybe_load_json(manifest_path)
305
+ if payload and str(payload.get("metadata_jsonl")) == str(args.metadata_jsonl):
306
+ count = int(payload.get("count") or 0)
307
+ if count > 0:
308
+ return min(count, int(args.limit)) if args.limit > 0 else count
309
+ return None
310
+
311
+
312
+ def count_metadata_rows(args: argparse.Namespace) -> int:
313
+ _, metadata_path = resolve_metadata_context(args)
314
+ count = 0
315
+ with metadata_path.open("rb") as f:
316
+ for chunk in iter(lambda: f.read(1024 * 1024 * 16), b""):
317
+ count += chunk.count(b"\n")
318
+ if args.limit > 0:
319
+ return min(count, int(args.limit))
320
+ return count
321
+
322
+
323
+ def dense_source_key(row: dict) -> str:
324
+ for key in ("name", "id", "video_path"):
325
+ value = row.get(key)
326
+ if isinstance(value, str) and value:
327
+ return value.split(".chunk", 1)[0]
328
+ return ""
329
+
330
+
331
+ def cache_name_from_row(row: dict, args: argparse.Namespace) -> str:
332
+ video_value = row.get(args.video_path_key) or row.get("video_path") or row.get("original_video")
333
+ fps_tag = str(float(args.video_sample_fps)).replace(".", "p")
334
+ return f"{Path(str(video_value)).stem}.fps{fps_tag}.size{int(args.video_size_pixels)}.npz"
335
+
336
+
337
+ def compatible_resume_payload(
338
+ payload: dict | None,
339
+ args: argparse.Namespace,
340
+ embed_root: Path,
341
+ *,
342
+ shard_id: int,
343
+ shard_num: int,
344
+ ) -> bool:
345
+ if not payload:
346
+ return False
347
+ expected = {
348
+ "dataset_root": str(args.dataset_root),
349
+ "embed_root": str(embed_root),
350
+ "video_sample_fps": float(args.video_sample_fps),
351
+ "video_size_pixels": int(args.video_size_pixels),
352
+ "shard_id": int(shard_id),
353
+ "shard_num": int(shard_num),
354
+ "input_already_sharded": bool(args.input_already_sharded),
355
+ "direct_source_clips": bool(args.direct_source_clips),
356
+ }
357
+ return all(payload.get(key) == value for key, value in expected.items())
358
+
359
+
360
+ def parse_resume_state_name(path: Path) -> tuple[int, int] | None:
361
+ name = path.name
362
+ prefix = "resume.shard-"
363
+ if not name.startswith(prefix) or "-of-" not in name or not name.endswith(".json"):
364
+ return None
365
+ try:
366
+ left, right = name[len(prefix):-len(".json")].split("-of-", 1)
367
+ return int(left), int(right)
368
+ except ValueError:
369
+ return None
370
+
371
+
372
+ def infer_frontier_from_parent_resume(args: argparse.Namespace, embed_root: Path) -> int:
373
+ if args.shard_num <= 1:
374
+ return 0
375
+ dataset_root, _ = resolve_metadata_context(args)
376
+ candidates = []
377
+ for resume_path in sorted(embed_root.glob("resume.shard-*-of-*.json")):
378
+ parsed = parse_resume_state_name(resume_path)
379
+ if parsed is None:
380
+ continue
381
+ parent_id, parent_num = parsed
382
+ if parent_num == args.shard_num:
383
+ continue
384
+ payload = maybe_load_json(resume_path)
385
+ if not compatible_resume_payload(
386
+ payload,
387
+ args,
388
+ embed_root,
389
+ shard_id=parent_id,
390
+ shard_num=parent_num,
391
+ ):
392
+ continue
393
+ parent_frontier = max(0, int(payload.get("frontier") or 0))
394
+ if parent_frontier <= 0:
395
+ continue
396
+ parent_metadata = dataset_root / str(payload.get("metadata_jsonl") or "")
397
+ if not parent_metadata.exists():
398
+ parent_metadata = dataset_root / f"metadata.shard-{parent_id:05d}-of-{parent_num:05d}.jsonl"
399
+ if not parent_metadata.exists():
400
+ continue
401
+ candidates.append((parent_num, parent_id, parent_frontier, parent_metadata))
402
+
403
+ if not candidates:
404
+ return 0
405
+
406
+ completed_cache_names = set()
407
+ for parent_num, parent_id, parent_frontier, parent_metadata in candidates:
408
+ with parent_metadata.open("r", encoding="utf-8") as f:
409
+ iterator = tqdm(
410
+ islice(f, parent_frontier),
411
+ total=parent_frontier,
412
+ desc=f"load resume {parent_id}/{parent_num}",
413
+ )
414
+ for line in iterator:
415
+ line = line.strip()
416
+ if not line:
417
+ continue
418
+ completed_cache_names.add(cache_name_from_row(json.loads(line), args))
419
+
420
+ inherited = 0
421
+ for video_task in tqdm(iter_metadata_tasks_stream(args), desc="inherit resume"):
422
+ cache_name = video_embed_cache_path(
423
+ Path(video_task["video_path"]),
424
+ embed_root,
425
+ video_sample_fps=args.video_sample_fps,
426
+ video_size_pixels=args.video_size_pixels,
427
+ ).name
428
+ if cache_name not in completed_cache_names:
429
+ break
430
+ inherited = int(video_task["task_index"]) + 1
431
+
432
+ print(
433
+ f"[RESUME] inherited frontier={inherited} from "
434
+ f"{len(candidates)} compatible previous shard resume file(s)"
435
+ )
436
+ return inherited
437
+
438
+
439
+ def write_manifest(
440
+ args: argparse.Namespace,
441
+ embed_root: Path,
442
+ total_video_paths: int,
443
+ ) -> None:
444
+ manifest = {
445
+ "video_root": str(args.video_root),
446
+ "dataset_root": str(args.dataset_root),
447
+ "metadata_jsonl": str(args.metadata_jsonl),
448
+ "embed_root": str(embed_root),
449
+ "model_path": str(args.model_path),
450
+ "video_sample_fps": float(args.video_sample_fps),
451
+ "video_size_pixels": int(args.video_size_pixels),
452
+ "video_decoder": str(args.video_decoder),
453
+ "dtype": args.dtype,
454
+ "num_workers": int(args.num_workers),
455
+ "save_compressed": bool(args.save_compressed),
456
+ "existing_cache_check": str(args.existing_cache_check),
457
+ "count": int(total_video_paths),
458
+ "total_count": int(total_video_paths),
459
+ "shard_id": int(args.shard_id),
460
+ "shard_num": int(args.shard_num),
461
+ "input_already_sharded": bool(args.input_already_sharded),
462
+ }
463
+ manifest_path = (
464
+ shard_manifest_path(embed_root, args.shard_id, args.shard_num)
465
+ if args.shard_num > 1
466
+ else embed_root / "manifest.json"
467
+ )
468
+ write_json_atomic(manifest_path, manifest)
469
+ if args.shard_num > 1:
470
+ merge_shard_manifests(embed_root, args.shard_num)
471
+
472
+
473
+ def fast_resume_enabled(args: argparse.Namespace) -> bool:
474
+ return bool(args.metadata_jsonl and args.input_already_sharded and args.direct_source_clips)
475
+
476
+
477
+ def load_resume_frontier(args: argparse.Namespace, embed_root: Path) -> tuple[int, int | None, Path]:
478
+ state_path = resume_state_path(embed_root, args.shard_id, args.shard_num)
479
+ payload = maybe_load_json(state_path)
480
+ if not compatible_resume_payload(
481
+ payload,
482
+ args,
483
+ embed_root,
484
+ shard_id=args.shard_id,
485
+ shard_num=args.shard_num,
486
+ ) or not payload or payload.get("metadata_jsonl") != str(args.metadata_jsonl):
487
+ return infer_frontier_from_parent_resume(args, embed_root), None, state_path
488
+ frontier = max(0, int(payload.get("frontier") or 0))
489
+ total_count = int(payload.get("total_count") or 0) or None
490
+ if args.limit > 0:
491
+ frontier = min(frontier, int(args.limit))
492
+ if total_count is not None:
493
+ total_count = min(total_count, int(args.limit))
494
+ return frontier, total_count, state_path
495
+
496
+
497
+ class ResumeTracker:
498
+ def __init__(
499
+ self,
500
+ *,
501
+ path: Path,
502
+ args: argparse.Namespace,
503
+ embed_root: Path,
504
+ frontier: int,
505
+ total_count: int | None,
506
+ ) -> None:
507
+ self.path = path
508
+ self.args = args
509
+ self.embed_root = embed_root
510
+ self.frontier = max(0, int(frontier))
511
+ self.total_count = int(total_count) if total_count is not None else None
512
+ self._completed: set[int] = set()
513
+ self._dirty = True
514
+ self._last_flush_time = 0.0
515
+
516
+ def mark_complete(self, task_index: int | None) -> None:
517
+ if task_index is None or task_index < 0 or task_index < self.frontier:
518
+ return
519
+ self._completed.add(int(task_index))
520
+ advanced = False
521
+ while self.frontier in self._completed:
522
+ self._completed.remove(self.frontier)
523
+ self.frontier += 1
524
+ advanced = True
525
+ if advanced:
526
+ self._dirty = True
527
+
528
+ def maybe_set_total_count(self, total_count: int) -> None:
529
+ total_count = int(total_count)
530
+ if self.total_count != total_count:
531
+ self.total_count = total_count
532
+ self._dirty = True
533
+
534
+ def flush(self, force: bool = False) -> None:
535
+ now = time.time()
536
+ if not force and (not self._dirty or now - self._last_flush_time < 10.0):
537
+ return
538
+ payload = {
539
+ "dataset_root": str(self.args.dataset_root),
540
+ "metadata_jsonl": str(self.args.metadata_jsonl),
541
+ "embed_root": str(self.embed_root),
542
+ "video_sample_fps": float(self.args.video_sample_fps),
543
+ "video_size_pixels": int(self.args.video_size_pixels),
544
+ "shard_id": int(self.args.shard_id),
545
+ "shard_num": int(self.args.shard_num),
546
+ "input_already_sharded": bool(self.args.input_already_sharded),
547
+ "direct_source_clips": bool(self.args.direct_source_clips),
548
+ "frontier": int(self.frontier),
549
+ "total_count": int(self.total_count) if self.total_count is not None else None,
550
+ "updated_at": now,
551
+ }
552
+ write_json_atomic(self.path, payload)
553
+ self._dirty = False
554
+ self._last_flush_time = now
555
+
556
+
557
+ def extend_resume_frontier(
558
+ args: argparse.Namespace,
559
+ embed_root: Path,
560
+ frontier: int,
561
+ ) -> int:
562
+ if args.overwrite:
563
+ return 0
564
+ current_frontier = max(0, int(frontier))
565
+ check_workers = max(1, int(os.environ.get("RESUME_SCAN_NUM_WORKERS", "32")))
566
+ batch_size = max(check_workers, int(os.environ.get("RESUME_SCAN_BATCH_SIZE", str(check_workers * 8))))
567
+ iterator = iter_metadata_tasks_stream(args, start_index=current_frontier)
568
+ progress = tqdm(desc=f"resume frontier x{check_workers}")
569
+
570
+ def check_task(video_task: dict) -> tuple[int, bool, Path]:
571
+ video_path = Path(video_task["video_path"])
572
+ cache_path = video_embed_cache_path(
573
+ video_path,
574
+ embed_root,
575
+ video_sample_fps=args.video_sample_fps,
576
+ video_size_pixels=args.video_size_pixels,
577
+ )
578
+ return (
579
+ int(video_task["task_index"]),
580
+ is_existing_cache_ready_fast(cache_path, args.existing_cache_check),
581
+ cache_path,
582
+ )
583
+
584
+ with ThreadPoolExecutor(max_workers=check_workers) as executor:
585
+ while True:
586
+ batch = list(islice(iterator, batch_size))
587
+ if not batch:
588
+ break
589
+ results = list(executor.map(check_task, batch))
590
+ for task_index, ready, cache_path in results:
591
+ progress.update(1)
592
+ if ready:
593
+ current_frontier = task_index + 1
594
+ continue
595
+ if cache_path.exists():
596
+ remove_path(cache_path)
597
+ progress.close()
598
+ return current_frontier
599
+ progress.close()
600
+ return current_frontier
601
+
602
+
603
+ def acquire_lock(lock_path: Path) -> int | None:
604
+ lock_payload = {
605
+ "pid": os.getpid(),
606
+ "host": socket.gethostname(),
607
+ "time": time.time(),
608
+ }
609
+ while True:
610
+ try:
611
+ fd = os.open(str(lock_path), os.O_CREAT | os.O_EXCL | os.O_WRONLY)
612
+ os.write(fd, json.dumps(lock_payload).encode("utf-8"))
613
+ return fd
614
+ except FileExistsError:
615
+ try:
616
+ payload = json.loads(lock_path.read_text(encoding="utf-8"))
617
+ pid = int(payload.get("pid") or -1)
618
+ except Exception:
619
+ pid = -1
620
+ if pid <= 0 or not Path(f"/proc/{pid}").exists():
621
+ remove_path(lock_path)
622
+ continue
623
+ return None
624
+
625
+
626
+ def release_lock(fd: int | None, lock_path: Path) -> None:
627
+ if fd is not None:
628
+ try:
629
+ os.close(fd)
630
+ except OSError:
631
+ pass
632
+ remove_path(lock_path)
633
+
634
+
635
+ def init_worker(model_path: str) -> None:
636
+ global _WORKER_PROCESSOR, _WORKER_MODEL_PATH
637
+ _WORKER_MODEL_PATH = model_path
638
+ _WORKER_PROCESSOR = Qwen2_5OmniProcessor.from_pretrained(model_path, use_fast=False)
639
+
640
+
641
+ def get_worker_processor() -> Qwen2_5OmniProcessor:
642
+ global _WORKER_PROCESSOR
643
+ if _WORKER_PROCESSOR is None:
644
+ if not _WORKER_MODEL_PATH:
645
+ raise RuntimeError("Worker processor is not initialized")
646
+ _WORKER_PROCESSOR = Qwen2_5OmniProcessor.from_pretrained(_WORKER_MODEL_PATH, use_fast=False)
647
+ return _WORKER_PROCESSOR
648
+
649
+
650
+ def cache_one_video(task: dict) -> dict:
651
+ video_path = Path(task["video_path"])
652
+ decode_video_path = Path(task.get("decode_video_path") or video_path)
653
+ embed_root = Path(task["embed_root"])
654
+ cache_path = video_embed_cache_path(
655
+ video_path,
656
+ embed_root,
657
+ video_sample_fps=float(task["video_sample_fps"]),
658
+ video_size_pixels=int(task["video_size_pixels"]),
659
+ )
660
+ overwrite = bool(task["overwrite"])
661
+ existing_cache_check = str(task.get("existing_cache_check", "strict"))
662
+ task_index = int(task.get("task_index", -1))
663
+ result_base = {"path": str(cache_path), "task_index": task_index}
664
+
665
+ if cache_path.exists() and not overwrite:
666
+ if is_existing_cache_ready(cache_path, existing_cache_check):
667
+ return {"status": "skipped", **result_base}
668
+ remove_path(cache_path)
669
+
670
+ lock_path = cache_path.with_name(f"{cache_path.name}.lock")
671
+ lock_fd = acquire_lock(lock_path)
672
+ if lock_fd is None:
673
+ if cache_path.exists() and is_existing_cache_ready(cache_path, existing_cache_check):
674
+ return {"status": "skipped", **result_base}
675
+ return {"status": "locked", **result_base}
676
+
677
+ tmp_path = cache_path.with_name(f".{cache_path.name}.{os.getpid()}.{uuid.uuid4().hex}.tmp")
678
+ try:
679
+ if cache_path.exists() and not overwrite:
680
+ if is_existing_cache_ready(cache_path, existing_cache_check):
681
+ return {"status": "skipped", **result_base}
682
+ remove_path(cache_path)
683
+
684
+ if bool(task.get("direct_source_clip", False)):
685
+ video, source_video_fps, video_duration_sec = load_video_clip_array(
686
+ decode_video_path,
687
+ start_sec=float(task["clip_start_sec"]),
688
+ duration_sec=float(task["clip_duration_sec"]),
689
+ video_sample_fps=float(task["video_sample_fps"]),
690
+ video_size_pixels=int(task["video_size_pixels"]),
691
+ video_decoder=str(task["video_decoder"]),
692
+ )
693
+ else:
694
+ video, source_video_fps, video_duration_sec = load_video_array(
695
+ decode_video_path,
696
+ video_sample_fps=float(task["video_sample_fps"]),
697
+ video_size_pixels=int(task["video_size_pixels"]),
698
+ video_decoder=str(task["video_decoder"]),
699
+ )
700
+ processor = get_worker_processor()
701
+ video_inputs = processor.video_processor(
702
+ videos=[video],
703
+ size={
704
+ "shortest_edge": int(task["video_size_pixels"]),
705
+ "longest_edge": int(task["video_size_pixels"]),
706
+ },
707
+ )
708
+ video_second_per_grid = np.asarray(
709
+ [float(processor.video_processor.temporal_patch_size) / float(task["video_sample_fps"])],
710
+ dtype=np.float32,
711
+ )
712
+ dtype = np.float16 if str(task["dtype"]) == "float16" else np.float32
713
+ payload = {
714
+ "pixel_values_videos": np.asarray(video_inputs["pixel_values_videos"], dtype=dtype),
715
+ "video_grid_thw": np.asarray(video_inputs["video_grid_thw"], dtype=np.int64),
716
+ "video_second_per_grid": video_second_per_grid,
717
+ "video_sample_fps": np.asarray([float(task["video_sample_fps"])], dtype=np.float32),
718
+ "video_size_pixels": np.asarray([int(task["video_size_pixels"])], dtype=np.int64),
719
+ "source_video_fps": np.asarray([float(source_video_fps)], dtype=np.float32),
720
+ "video_duration_sec": np.asarray([float(video_duration_sec)], dtype=np.float32),
721
+ }
722
+ with tmp_path.open("wb") as f:
723
+ if bool(task["save_compressed"]):
724
+ np.savez_compressed(f, **payload)
725
+ else:
726
+ np.savez(f, **payload)
727
+ if not is_valid_cache(tmp_path):
728
+ raise RuntimeError(f"Invalid cache was written: {tmp_path}")
729
+ tmp_path.replace(cache_path)
730
+ return {"status": "cached", **result_base}
731
+ except Exception as exc:
732
+ remove_path(tmp_path)
733
+ return {"status": "failed", "error": str(exc), **result_base}
734
+ finally:
735
+ release_lock(lock_fd, lock_path)
736
+
737
+
738
+ def build_cache_task(args: argparse.Namespace, video_task: dict, embed_root: Path) -> dict:
739
+ task = {
740
+ "video_path": str(video_task["video_path"]),
741
+ "decode_video_path": str(video_task.get("decode_video_path") or video_task["video_path"]),
742
+ "embed_root": str(embed_root),
743
+ "video_sample_fps": float(args.video_sample_fps),
744
+ "video_size_pixels": int(args.video_size_pixels),
745
+ "video_decoder": str(args.video_decoder),
746
+ "dtype": str(args.dtype),
747
+ "save_compressed": bool(args.save_compressed),
748
+ "overwrite": bool(args.overwrite),
749
+ "existing_cache_check": str(args.existing_cache_check),
750
+ }
751
+ if "clip_start_sec" in video_task:
752
+ task.update(
753
+ {
754
+ "direct_source_clip": True,
755
+ "clip_start_sec": float(video_task["clip_start_sec"]),
756
+ "clip_duration_sec": float(video_task["clip_duration_sec"]),
757
+ }
758
+ )
759
+ if "task_index" in video_task:
760
+ task["task_index"] = int(video_task["task_index"])
761
+ return task
762
+
763
+
764
+ def update_result_counters(
765
+ result: dict,
766
+ tracker: ResumeTracker,
767
+ counters: dict[str, int],
768
+ ) -> None:
769
+ status = result["status"]
770
+ task_index = result.get("task_index")
771
+ if status == "cached":
772
+ counters["cached"] += 1
773
+ tracker.mark_complete(task_index)
774
+ elif status == "skipped":
775
+ counters["skipped"] += 1
776
+ tracker.mark_complete(task_index)
777
+ elif status == "locked":
778
+ counters["locked"] += 1
779
+ else:
780
+ counters["failed"] += 1
781
+ print(f"[FAILED] {result['path']}: {result.get('error', '')}")
782
+ tracker.flush()
783
+
784
+
785
+ def run_fast_resume_cache(args: argparse.Namespace, embed_root: Path) -> None:
786
+ frontier, known_total_count, state_path = load_resume_frontier(args, embed_root)
787
+ if known_total_count is None:
788
+ known_total_count = load_known_total_count(args, embed_root)
789
+ if known_total_count is None:
790
+ known_total_count = count_metadata_rows(args)
791
+ write_manifest(args, embed_root, known_total_count)
792
+ extended_frontier = extend_resume_frontier(args, embed_root, frontier)
793
+ tracker = ResumeTracker(
794
+ path=state_path,
795
+ args=args,
796
+ embed_root=embed_root,
797
+ frontier=extended_frontier,
798
+ total_count=known_total_count,
799
+ )
800
+ tracker.flush(force=True)
801
+
802
+ if known_total_count is not None:
803
+ print(
804
+ f"[SHARD] cache shard={args.shard_id}/{args.shard_num} mode=pre-sharded "
805
+ f"resume={extended_frontier}/{known_total_count}"
806
+ )
807
+ write_manifest(args, embed_root, known_total_count)
808
+ else:
809
+ print(
810
+ f"[SHARD] cache shard={args.shard_id}/{args.shard_num} mode=pre-sharded "
811
+ f"resume={extended_frontier}/?"
812
+ )
813
+
814
+ counters = {
815
+ "cached": 0,
816
+ "skipped": 0,
817
+ "locked": 0,
818
+ "failed": 0,
819
+ "invalid": 0,
820
+ }
821
+ total_hint = None
822
+ if tracker.total_count is not None:
823
+ total_hint = max(0, int(tracker.total_count) - int(extended_frontier))
824
+
825
+ if args.num_workers == 1:
826
+ progress = tqdm(total=total_hint, desc="cache omni video")
827
+ total_video_paths = extended_frontier
828
+ for video_task in iter_metadata_tasks_stream(args, start_index=extended_frontier):
829
+ total_video_paths = int(video_task["task_index"]) + 1
830
+ task = build_cache_task(args, video_task, embed_root)
831
+ result = cache_one_video(task)
832
+ update_result_counters(result, tracker, counters)
833
+ progress.update(1)
834
+ progress.close()
835
+ else:
836
+ max_in_flight = max(int(args.num_workers) * 4, int(args.num_workers))
837
+ progress = tqdm(total=total_hint, desc=f"cache omni video x{args.num_workers}")
838
+ total_video_paths = extended_frontier
839
+ with ProcessPoolExecutor(
840
+ max_workers=args.num_workers,
841
+ initializer=init_worker,
842
+ initargs=(args.model_path,),
843
+ ) as executor:
844
+ pending_futures: dict = {}
845
+ task_iterator = iter_metadata_tasks_stream(args, start_index=extended_frontier)
846
+ exhausted = False
847
+ while True:
848
+ while not exhausted and len(pending_futures) < max_in_flight:
849
+ try:
850
+ video_task = next(task_iterator)
851
+ except StopIteration:
852
+ exhausted = True
853
+ break
854
+ total_video_paths = int(video_task["task_index"]) + 1
855
+ task = build_cache_task(args, video_task, embed_root)
856
+ future = executor.submit(cache_one_video, task)
857
+ pending_futures[future] = None
858
+
859
+ if not pending_futures:
860
+ break
861
+
862
+ done, _ = wait(tuple(pending_futures.keys()), return_when=FIRST_COMPLETED)
863
+ for future in done:
864
+ pending_futures.pop(future, None)
865
+ result = future.result()
866
+ update_result_counters(result, tracker, counters)
867
+ progress.update(1)
868
+ progress.close()
869
+
870
+ if tracker.total_count is None:
871
+ tracker.maybe_set_total_count(total_video_paths)
872
+ else:
873
+ tracker.maybe_set_total_count(max(int(tracker.total_count), int(total_video_paths)))
874
+ tracker.flush(force=True)
875
+ if total_video_paths > 0:
876
+ write_manifest(args, embed_root, total_video_paths if tracker.total_count is None else int(tracker.total_count))
877
+
878
+ print(
879
+ f"cached={counters['cached']} skipped={counters['skipped']} invalid_removed={counters['invalid']} "
880
+ f"locked={counters['locked']} failed={counters['failed']} embed_root={embed_root}"
881
+ )
882
+ if counters["failed"] > 0:
883
+ raise SystemExit(1)
884
+ if counters["locked"] > 0:
885
+ raise SystemExit(2)
886
+
887
+
888
+ def main() -> None:
889
+ args = parse_args()
890
+ if args.num_workers <= 0:
891
+ raise ValueError("--num-workers must be positive")
892
+ if args.shard_num <= 0:
893
+ raise ValueError("--shard-num must be positive")
894
+ if args.shard_id < 0 or args.shard_id >= args.shard_num:
895
+ raise ValueError("--shard-id must satisfy 0 <= shard_id < shard_num")
896
+
897
+ embed_root = Path(args.embed_root)
898
+ embed_root.mkdir(parents=True, exist_ok=True)
899
+
900
+ if fast_resume_enabled(args):
901
+ run_fast_resume_cache(args, embed_root)
902
+ return
903
+
904
+ video_tasks = iter_video_tasks(args)
905
+ if args.limit > 0:
906
+ video_tasks = video_tasks[: args.limit]
907
+ total_video_paths = len(video_tasks)
908
+ if args.shard_num > 1 and not args.input_already_sharded:
909
+ video_tasks = [
910
+ video_task for video_task in video_tasks
911
+ if stable_hash_mod(str(video_task["video_path"]), args.shard_num) == args.shard_id
912
+ ]
913
+ if args.shard_num > 1:
914
+ mode = "pre-sharded" if args.input_already_sharded else "hash"
915
+ print(
916
+ f"[SHARD] cache shard={args.shard_id}/{args.shard_num} mode={mode} "
917
+ f"videos={len(video_tasks)}/{total_video_paths}"
918
+ )
919
+
920
+ write_manifest(args, embed_root, total_video_paths)
921
+
922
+ pending_tasks = []
923
+ cached = 0
924
+ skipped = 0
925
+ locked = 0
926
+ failed = 0
927
+ invalid = 0
928
+ scan_progress = tqdm(video_tasks, total=len(video_tasks), desc="scan cache state")
929
+ for video_task in scan_progress:
930
+ video_path = Path(video_task["video_path"])
931
+ cache_path = video_embed_cache_path(
932
+ video_path,
933
+ embed_root,
934
+ video_sample_fps=args.video_sample_fps,
935
+ video_size_pixels=args.video_size_pixels,
936
+ )
937
+ if cache_path.exists() and not args.overwrite:
938
+ if is_existing_cache_ready(cache_path, args.existing_cache_check):
939
+ skipped += 1
940
+ continue
941
+ remove_path(cache_path)
942
+ invalid += 1
943
+ pending_tasks.append(build_cache_task(args, video_task, embed_root))
944
+
945
+ if pending_tasks:
946
+ if args.num_workers == 1:
947
+ init_worker(args.model_path)
948
+ iterator = (cache_one_video(task) for task in pending_tasks)
949
+ progress = tqdm(iterator, total=len(pending_tasks), desc="cache omni video")
950
+ for result in progress:
951
+ status = result["status"]
952
+ if status == "cached":
953
+ cached += 1
954
+ elif status == "skipped":
955
+ skipped += 1
956
+ elif status == "locked":
957
+ locked += 1
958
+ else:
959
+ failed += 1
960
+ print(f"[FAILED] {result['path']}: {result.get('error', '')}")
961
+ else:
962
+ with ProcessPoolExecutor(
963
+ max_workers=args.num_workers,
964
+ initializer=init_worker,
965
+ initargs=(args.model_path,),
966
+ ) as executor:
967
+ futures = [executor.submit(cache_one_video, task) for task in pending_tasks]
968
+ for future in tqdm(
969
+ as_completed(futures),
970
+ total=len(futures),
971
+ desc=f"cache omni video x{args.num_workers}",
972
+ ):
973
+ result = future.result()
974
+ status = result["status"]
975
+ if status == "cached":
976
+ cached += 1
977
+ elif status == "skipped":
978
+ skipped += 1
979
+ elif status == "locked":
980
+ locked += 1
981
+ else:
982
+ failed += 1
983
+ print(f"[FAILED] {result['path']}: {result.get('error', '')}")
984
+
985
+ print(
986
+ f"cached={cached} skipped={skipped} invalid_removed={invalid} "
987
+ f"locked={locked} failed={failed} embed_root={embed_root}"
988
+ )
989
+ if failed > 0:
990
+ raise SystemExit(1)
991
+ if locked > 0:
992
+ raise SystemExit(2)
993
+
994
+
995
+ if __name__ == "__main__":
996
+ main()
cache_video_omni_embeds.sh ADDED
@@ -0,0 +1,275 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/usr/bin/env bash
2
+ set -euo pipefail
3
+
4
+ cd /cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/Psi0-yth-v18
5
+ PSI_PYTHON_BIN=${PSI_PYTHON_BIN:-}
6
+ PSI_PYTHON_ENV=${PSI_PYTHON_ENV:-}
7
+ if [ -z "$PSI_PYTHON_BIN" ] && [ -n "$PSI_PYTHON_ENV" ]; then
8
+ if [ ! -x "$PSI_PYTHON_ENV/bin/python" ]; then
9
+ echo "ERROR: PSI_PYTHON_ENV does not contain an executable python: $PSI_PYTHON_ENV/bin/python" >&2
10
+ exit 2
11
+ fi
12
+ source "$PSI_PYTHON_ENV/bin/activate"
13
+ fi
14
+ [ -f .env ] && source .env
15
+
16
+ export PYTHONPATH=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/Psi0-yth-v18/src
17
+ export CUDA_VISIBLE_DEVICES=${CUDA_VISIBLE_DEVICES:-0}
18
+ export OMP_NUM_THREADS=${OMP_NUM_THREADS:-1}
19
+ PYTHON_BIN=${PSI_PYTHON_BIN:-$(command -v python)}
20
+ if [ ! -x "$PYTHON_BIN" ]; then
21
+ echo "ERROR: python executable is not valid: $PYTHON_BIN" >&2
22
+ exit 2
23
+ fi
24
+
25
+ DATASET_ROOT=${PSI_VIDEO_CLIP_DATASET_ROOT:-/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/videogen/psi0_video_clip_50f_rand8}
26
+ DANCE_CLIP_ROOT=$DATASET_ROOT/dance_1723_video_10s
27
+ MOTION_EXAMPLE_CLIP_ROOT=$DATASET_ROOT/motion_example_0506_demo
28
+ DANCE_CLIP_EMBED_ROOT=${PSI_DANCE_VIDEO_CLIP_EMBED_ROOT:-$DANCE_CLIP_ROOT/embeds}
29
+ MOTION_EXAMPLE_CLIP_EMBED_ROOT=${PSI_MOTION_EXAMPLE_VIDEO_CLIP_EMBED_ROOT:-$MOTION_EXAMPLE_CLIP_ROOT/embeds}
30
+
31
+ DANCE_MOTION_ROOT=${PSI_DANCE_MOTION_ROOT:-/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/GR00T-WholeBodyControl/gear_sonic_deploy/reference/converted/psi0_dance/gvhmr}
32
+ DANCE_VIDEO_ROOT=${PSI_DANCE_VIDEO_ROOT:-/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/dance_1723_video_psi0/dance_1723_video_10s}
33
+ MOTION_EXAMPLE_ROOT=${PSI_VIDEO_DEMO_ROOT:-/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/videogen/motion_example_0506_demo}
34
+
35
+ MODEL_PATH=${PSI_PRETRAIN_BACKBONE_PATH:-/cpfs01/VideoGen/public_ckpt/Qwen/Qwen2.5-Omni-3B}
36
+ VIDEO_SAMPLE_FPS=${VIDEO_SAMPLE_FPS:-32}
37
+ VIDEO_SIZE_PIXELS=${VIDEO_SIZE_PIXELS:-50176}
38
+ VIDEO_DECODER=${VIDEO_DECODER:-ffmpeg}
39
+ CLIP_FRAMES=${CLIP_FRAMES:-50}
40
+ ACTION_CHUNK_FRAMES=${ACTION_CHUNK_FRAMES:-30}
41
+ CHUNK_STRIDE_FRAMES=${CHUNK_STRIDE_FRAMES:-20}
42
+ MOTION_FPS=${MOTION_FPS:-50}
43
+ CLIPS_PER_VIDEO=${CLIPS_PER_VIDEO:-2}
44
+ CLIP_SELECTION_MODE=${CLIP_SELECTION_MODE:-middle_random}
45
+ CLIP_RANDOM_SEED=${CLIP_RANDOM_SEED:-42}
46
+ CLIP_MIDDLE_FRACTION=${CLIP_MIDDLE_FRACTION:-0.8}
47
+ CHUNK_PROGRESS_INTERVAL=${CHUNK_PROGRESS_INTERVAL:-0}
48
+ BUILD_NUM_WORKERS=${BUILD_NUM_WORKERS:-32}
49
+ CHUNK_NUM_WORKERS=${CHUNK_NUM_WORKERS:-1}
50
+ FFMPEG_THREADS=${FFMPEG_THREADS:-1}
51
+ X264_PRESET=${X264_PRESET:-}
52
+ EXISTING_VIDEO_CHECK=${EXISTING_VIDEO_CHECK:-ffprobe}
53
+ EMBED_NUM_WORKERS=${EMBED_NUM_WORKERS:-64}
54
+ EMBED_SAVE_COMPRESSED=${EMBED_SAVE_COMPRESSED:-0}
55
+ EXISTING_CACHE_CHECK=${EXISTING_CACHE_CHECK:-strict}
56
+ BUILD_CLIPS=${BUILD_CLIPS:-1}
57
+ CACHE_EMBEDS=${CACHE_EMBEDS:-1}
58
+ BUILD_DANCE=${BUILD_DANCE:-1}
59
+ BUILD_MOTION_EXAMPLE=${BUILD_MOTION_EXAMPLE:-1}
60
+ CACHE_DANCE=${CACHE_DANCE:-1}
61
+ CACHE_MOTION_EXAMPLE=${CACHE_MOTION_EXAMPLE:-1}
62
+ LIMIT=${LIMIT:-0}
63
+ OVERWRITE_CLIPS=${OVERWRITE_CLIPS:-0}
64
+ OVERWRITE_EMBEDS=${OVERWRITE_EMBEDS:-0}
65
+ SHARD_NUM=${SHARD_NUM:-1}
66
+ SHARD_ID=${SHARD_ID:-0}
67
+ METADATA_ONLY=${METADATA_ONLY:-0}
68
+ DIRECT_SOURCE_CLIPS=${DIRECT_SOURCE_CLIPS:-0}
69
+
70
+ if ! [[ "$SHARD_NUM" =~ ^[0-9]+$ ]] || ! [[ "$SHARD_ID" =~ ^[0-9]+$ ]]; then
71
+ echo "SHARD_NUM and SHARD_ID must be non-negative integers" >&2
72
+ exit 2
73
+ fi
74
+ if (( SHARD_NUM <= 0 )); then
75
+ echo "SHARD_NUM must be positive" >&2
76
+ exit 2
77
+ fi
78
+ if (( SHARD_ID >= SHARD_NUM )); then
79
+ echo "SHARD_ID must satisfy 0 <= SHARD_ID < SHARD_NUM" >&2
80
+ exit 2
81
+ fi
82
+ printf -v SHARD_ID_PAD "%05d" "$SHARD_ID"
83
+ printf -v SHARD_NUM_PAD "%05d" "$SHARD_NUM"
84
+
85
+ clip_overwrite_args=()
86
+ if [ "$OVERWRITE_CLIPS" = "1" ]; then
87
+ clip_overwrite_args+=(--overwrite)
88
+ fi
89
+
90
+ embed_overwrite_args=()
91
+ if [ "$OVERWRITE_EMBEDS" = "1" ]; then
92
+ embed_overwrite_args+=(--overwrite)
93
+ fi
94
+
95
+ embed_compress_args=()
96
+ if [ "$EMBED_SAVE_COMPRESSED" = "1" ]; then
97
+ embed_compress_args+=(--save-compressed)
98
+ fi
99
+
100
+ limit_args=()
101
+ if [ "$LIMIT" -gt 0 ]; then
102
+ limit_args+=(--limit "$LIMIT")
103
+ fi
104
+
105
+ shard_args=()
106
+ if (( SHARD_NUM > 1 )); then
107
+ shard_args+=(--shard-num "$SHARD_NUM" --shard-id "$SHARD_ID")
108
+ fi
109
+
110
+ chunk_progress_args=()
111
+ if [ "$CHUNK_PROGRESS_INTERVAL" -gt 0 ]; then
112
+ chunk_progress_args+=(--chunk-progress-interval "$CHUNK_PROGRESS_INTERVAL")
113
+ fi
114
+
115
+ metadata_only_args=()
116
+ if [ "$METADATA_ONLY" = "1" ]; then
117
+ metadata_only_args+=(--metadata-only)
118
+ fi
119
+
120
+ direct_source_clip_args=()
121
+ if [ "$DIRECT_SOURCE_CLIPS" = "1" ]; then
122
+ direct_source_clip_args+=(--direct-source-clips)
123
+ fi
124
+
125
+ cache_metadata_jsonl() {
126
+ local clip_root="$1"
127
+ local shard_metadata="metadata.shard-${SHARD_ID_PAD}-of-${SHARD_NUM_PAD}.jsonl"
128
+ if (( SHARD_NUM > 1 )) && [ -f "$clip_root/$shard_metadata" ]; then
129
+ printf "%s" "$shard_metadata"
130
+ else
131
+ printf "%s" "metadata.jsonl"
132
+ fi
133
+ }
134
+
135
+ echo "Dataset root: $DATASET_ROOT"
136
+ echo "Dance clip embed root: $DANCE_CLIP_EMBED_ROOT"
137
+ echo "Motion example clip embed root: $MOTION_EXAMPLE_CLIP_EMBED_ROOT"
138
+ echo "Build clips: $BUILD_CLIPS"
139
+ echo "Cache embeds: $CACHE_EMBEDS"
140
+ echo "Build dance: $BUILD_DANCE"
141
+ echo "Build motion_example: $BUILD_MOTION_EXAMPLE"
142
+ echo "Cache dance: $CACHE_DANCE"
143
+ echo "Cache motion_example: $CACHE_MOTION_EXAMPLE"
144
+ echo "Clip frames: $CLIP_FRAMES"
145
+ echo "Action chunk frames: $ACTION_CHUNK_FRAMES"
146
+ echo "Chunk stride frames: $CHUNK_STRIDE_FRAMES"
147
+ echo "Clips per video: $CLIPS_PER_VIDEO"
148
+ echo "Clip selection mode: $CLIP_SELECTION_MODE"
149
+ echo "Clip random seed: $CLIP_RANDOM_SEED"
150
+ echo "Clip middle fraction: $CLIP_MIDDLE_FRACTION"
151
+ echo "Chunk progress interval: $CHUNK_PROGRESS_INTERVAL"
152
+ echo "Build workers: $BUILD_NUM_WORKERS"
153
+ echo "Chunk workers per source: $CHUNK_NUM_WORKERS"
154
+ echo "FFmpeg threads: $FFMPEG_THREADS"
155
+ echo "x264 preset: ${X264_PRESET:-<default>}"
156
+ echo "Existing video check: $EXISTING_VIDEO_CHECK"
157
+ echo "Embed workers: $EMBED_NUM_WORKERS"
158
+ echo "Embed compressed: $EMBED_SAVE_COMPRESSED"
159
+ echo "Existing cache check: $EXISTING_CACHE_CHECK"
160
+ echo "Video sample fps: $VIDEO_SAMPLE_FPS"
161
+ echo "Video size pixels: $VIDEO_SIZE_PIXELS"
162
+ echo "Shard: $SHARD_ID/$SHARD_NUM"
163
+ echo "Metadata only: $METADATA_ONLY"
164
+ echo "Direct source clips: $DIRECT_SOURCE_CLIPS"
165
+ echo "Python: $PYTHON_BIN"
166
+
167
+ if [ "$CACHE_EMBEDS" = "1" ]; then
168
+ "$PYTHON_BIN" - <<'PY'
169
+ import sys
170
+ from transformers import Qwen2_5OmniProcessor
171
+ print("Python executable:", sys.executable)
172
+ print("Qwen2_5OmniProcessor import: ok")
173
+ PY
174
+ fi
175
+
176
+ if [ "$BUILD_CLIPS" = "1" ] && [ "$BUILD_DANCE" = "1" ]; then
177
+ "$PYTHON_BIN" scripts/data/build_video_clip_dataset.py \
178
+ --dataset dance \
179
+ --output-root "$DANCE_CLIP_ROOT" \
180
+ --dance-motion-root "$DANCE_MOTION_ROOT" \
181
+ --dance-video-root "$DANCE_VIDEO_ROOT" \
182
+ --clip-frames "$CLIP_FRAMES" \
183
+ --action-chunk-frames "$ACTION_CHUNK_FRAMES" \
184
+ --chunk-stride-frames "$CHUNK_STRIDE_FRAMES" \
185
+ --motion-fps "$MOTION_FPS" \
186
+ --clips-per-video "$CLIPS_PER_VIDEO" \
187
+ --clip-selection-mode "$CLIP_SELECTION_MODE" \
188
+ --random-seed "$CLIP_RANDOM_SEED" \
189
+ --middle-fraction "$CLIP_MIDDLE_FRACTION" \
190
+ --num-workers "$BUILD_NUM_WORKERS" \
191
+ --chunk-workers "$CHUNK_NUM_WORKERS" \
192
+ --ffmpeg-threads "$FFMPEG_THREADS" \
193
+ --x264-preset "$X264_PRESET" \
194
+ --existing-video-check "$EXISTING_VIDEO_CHECK" \
195
+ "${shard_args[@]}" \
196
+ "${chunk_progress_args[@]}" \
197
+ "${limit_args[@]}" \
198
+ "${metadata_only_args[@]}" \
199
+ "${clip_overwrite_args[@]}"
200
+ fi
201
+
202
+ if [ "$BUILD_CLIPS" = "1" ] && [ "$BUILD_MOTION_EXAMPLE" = "1" ]; then
203
+ "$PYTHON_BIN" scripts/data/build_video_clip_dataset.py \
204
+ --dataset motion_example \
205
+ --output-root "$MOTION_EXAMPLE_CLIP_ROOT" \
206
+ --motion-example-root "$MOTION_EXAMPLE_ROOT" \
207
+ --clip-frames "$CLIP_FRAMES" \
208
+ --action-chunk-frames "$ACTION_CHUNK_FRAMES" \
209
+ --chunk-stride-frames "$CHUNK_STRIDE_FRAMES" \
210
+ --motion-fps "$MOTION_FPS" \
211
+ --clips-per-video "$CLIPS_PER_VIDEO" \
212
+ --clip-selection-mode "$CLIP_SELECTION_MODE" \
213
+ --random-seed "$CLIP_RANDOM_SEED" \
214
+ --middle-fraction "$CLIP_MIDDLE_FRACTION" \
215
+ --num-workers "$BUILD_NUM_WORKERS" \
216
+ --chunk-workers "$CHUNK_NUM_WORKERS" \
217
+ --ffmpeg-threads "$FFMPEG_THREADS" \
218
+ --x264-preset "$X264_PRESET" \
219
+ --existing-video-check "$EXISTING_VIDEO_CHECK" \
220
+ "${shard_args[@]}" \
221
+ "${chunk_progress_args[@]}" \
222
+ "${limit_args[@]}" \
223
+ "${metadata_only_args[@]}" \
224
+ "${clip_overwrite_args[@]}"
225
+ fi
226
+
227
+ if [ "$CACHE_EMBEDS" = "1" ] && [ "$CACHE_DANCE" = "1" ]; then
228
+ dance_metadata_jsonl=$(cache_metadata_jsonl "$DANCE_CLIP_ROOT")
229
+ dance_cache_shard_args=("${shard_args[@]}")
230
+ if (( SHARD_NUM > 1 )) && [ "$dance_metadata_jsonl" != "metadata.jsonl" ]; then
231
+ dance_cache_shard_args+=(--input-already-sharded)
232
+ fi
233
+ "$PYTHON_BIN" cache_video_omni_embeds.py \
234
+ --dataset-root "$DANCE_CLIP_ROOT" \
235
+ --metadata-jsonl "$dance_metadata_jsonl" \
236
+ --embed-root "$DANCE_CLIP_EMBED_ROOT" \
237
+ --model-path "$MODEL_PATH" \
238
+ --video-sample-fps "$VIDEO_SAMPLE_FPS" \
239
+ --video-size-pixels "$VIDEO_SIZE_PIXELS" \
240
+ --video-decoder "$VIDEO_DECODER" \
241
+ --dtype float16 \
242
+ --num-workers "$EMBED_NUM_WORKERS" \
243
+ --existing-cache-check "$EXISTING_CACHE_CHECK" \
244
+ "${dance_cache_shard_args[@]}" \
245
+ "${limit_args[@]}" \
246
+ "${direct_source_clip_args[@]}" \
247
+ "${embed_compress_args[@]}" \
248
+ "${embed_overwrite_args[@]}"
249
+ fi
250
+
251
+ if [ "$CACHE_EMBEDS" = "1" ] && [ "$CACHE_MOTION_EXAMPLE" = "1" ]; then
252
+ motion_metadata_jsonl=$(cache_metadata_jsonl "$MOTION_EXAMPLE_CLIP_ROOT")
253
+ motion_cache_shard_args=("${shard_args[@]}")
254
+ if (( SHARD_NUM > 1 )) && [ "$motion_metadata_jsonl" != "metadata.jsonl" ]; then
255
+ motion_cache_shard_args+=(--input-already-sharded)
256
+ fi
257
+ "$PYTHON_BIN" cache_video_omni_embeds.py \
258
+ --dataset-root "$MOTION_EXAMPLE_CLIP_ROOT" \
259
+ --metadata-jsonl "$motion_metadata_jsonl" \
260
+ --embed-root "$MOTION_EXAMPLE_CLIP_EMBED_ROOT" \
261
+ --model-path "$MODEL_PATH" \
262
+ --video-sample-fps "$VIDEO_SAMPLE_FPS" \
263
+ --video-size-pixels "$VIDEO_SIZE_PIXELS" \
264
+ --video-decoder "$VIDEO_DECODER" \
265
+ --dtype float16 \
266
+ --num-workers "$EMBED_NUM_WORKERS" \
267
+ --existing-cache-check "$EXISTING_CACHE_CHECK" \
268
+ "${motion_cache_shard_args[@]}" \
269
+ "${limit_args[@]}" \
270
+ "${direct_source_clip_args[@]}" \
271
+ "${embed_compress_args[@]}" \
272
+ "${embed_overwrite_args[@]}"
273
+ fi
274
+
275
+ echo "[DONE] video clip datasets and missing caches are ready."
cache_video_omni_embeds_dense_stride1_direct_full.sh ADDED
@@ -0,0 +1,523 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/usr/bin/env bash
2
+ set -euo pipefail
3
+
4
+ PROJECT_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/Psi0-yth-v18
5
+ SCRIPT_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/Psi0-yth-v21
6
+ cd "$PROJECT_ROOT"
7
+
8
+ DATASET_ROOT=${PSI_VIDEO_CLIP_DATASET_ROOT:-/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/videogen/psi0_video_clip_50f_dense_stride1_direct_full}
9
+ SHARD_NUM_TOTAL=${SHARD_NUM_TOTAL:-4}
10
+ DATASET_NAMES=${DATASET_NAMES:-all}
11
+ DEFAULT_PSI_PYTHON_ENV=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/Psi0-yth-v15/.venv-psi
12
+ KNOWN_DATASETS=(
13
+ dance_1723_video_10s
14
+ motion_example_0506_demo
15
+ 2264_with_audio
16
+ distilled_video_0226
17
+ )
18
+ DATASET_NAME_LIST=()
19
+
20
+ if [ -z "${PSI_PYTHON_BIN:-}" ] && [ -z "${PSI_PYTHON_ENV:-}" ] && [ -x "$DEFAULT_PSI_PYTHON_ENV/bin/python" ]; then
21
+ export PSI_PYTHON_ENV="$DEFAULT_PSI_PYTHON_ENV"
22
+ fi
23
+
24
+ prepare_python_env() {
25
+ PSI_PYTHON_BIN=${PSI_PYTHON_BIN:-}
26
+ PSI_PYTHON_ENV=${PSI_PYTHON_ENV:-}
27
+ if [ -z "$PSI_PYTHON_BIN" ] && [ -n "$PSI_PYTHON_ENV" ]; then
28
+ if [ ! -x "$PSI_PYTHON_ENV/bin/python" ]; then
29
+ echo "ERROR: PSI_PYTHON_ENV does not contain an executable python: $PSI_PYTHON_ENV/bin/python" >&2
30
+ exit 2
31
+ fi
32
+ # shellcheck disable=SC1090
33
+ source "$PSI_PYTHON_ENV/bin/activate"
34
+ fi
35
+ [ -f .env ] && source .env
36
+ export PYTHONPATH=$PROJECT_ROOT/src
37
+ PYTHON_BIN=${PSI_PYTHON_BIN:-$(command -v python)}
38
+ if [ ! -x "$PYTHON_BIN" ]; then
39
+ echo "ERROR: python executable is not valid: $PYTHON_BIN" >&2
40
+ exit 2
41
+ fi
42
+ }
43
+
44
+ metadata_jsonl_for_current_shard() {
45
+ if (( SHARD_NUM_TOTAL > 1 )); then
46
+ printf "metadata.shard-%05d-of-%05d.jsonl" "$SHARD_ID_VALUE" "$SHARD_NUM_TOTAL"
47
+ else
48
+ printf "metadata.jsonl"
49
+ fi
50
+ }
51
+
52
+ build_video_only_dense_metadata() {
53
+ local source_root="$1"
54
+ local output_root="$2"
55
+ local dataset_name="$3"
56
+ local source_metadata_jsonl="${4:-metadata.jsonl}"
57
+ local shard_metadata_jsonl
58
+ shard_metadata_jsonl=$(metadata_jsonl_for_current_shard)
59
+
60
+ mkdir -p "$output_root"
61
+ echo "------------------------------------------"
62
+ echo "Building dense metadata for: ${dataset_name}"
63
+ echo "Source root: ${source_root}"
64
+ echo "Output root: ${output_root}"
65
+ echo "Metadata: ${shard_metadata_jsonl}"
66
+ echo "Shard: ${SHARD_ID_VALUE}/${SHARD_NUM_TOTAL}"
67
+ echo "------------------------------------------"
68
+
69
+ "$PYTHON_BIN" - "$source_root" "$output_root" "$dataset_name" "$source_metadata_jsonl" "$shard_metadata_jsonl" <<'PY'
70
+ import concurrent.futures
71
+ import hashlib
72
+ import json
73
+ import os
74
+ import subprocess
75
+ import sys
76
+ import uuid
77
+ from pathlib import Path
78
+
79
+ source_root = Path(sys.argv[1])
80
+ output_root = Path(sys.argv[2])
81
+ dataset_name = sys.argv[3]
82
+ source_metadata_jsonl = sys.argv[4]
83
+ shard_metadata_jsonl = sys.argv[5]
84
+
85
+ shard_num = int(os.environ.get("SHARD_NUM", os.environ.get("SHARD_NUM_TOTAL", "1")))
86
+ shard_id = int(os.environ.get("SHARD_ID", os.environ.get("SHARD_ID_VALUE", "0")))
87
+ limit = int(os.environ.get("LIMIT", "0"))
88
+ motion_fps = float(os.environ.get("MOTION_FPS", "50"))
89
+ action_chunk_frames = int(os.environ.get("ACTION_CHUNK_FRAMES", "30"))
90
+ chunk_stride_frames = max(1, int(os.environ.get("CHUNK_STRIDE_FRAMES", "1")))
91
+ context_frames = int(os.environ.get("CLIP_FRAMES", "50"))
92
+ clip_selection_mode = os.environ.get("CLIP_SELECTION_MODE", "all")
93
+ clip_random_seed = int(os.environ.get("CLIP_RANDOM_SEED", "42"))
94
+ clip_middle_fraction = float(os.environ.get("CLIP_MIDDLE_FRACTION", "0.8"))
95
+ build_num_workers = max(1, int(os.environ.get("BUILD_NUM_WORKERS", "16")))
96
+ chunk_progress_interval = max(0, int(os.environ.get("CHUNK_PROGRESS_INTERVAL", "0")))
97
+ ffprobe_bin = os.environ.get("PSI_FFPROBE_BIN", "ffprobe")
98
+ default_caption = "Generate a motion sequence that matches the provided video."
99
+
100
+ metadata_path = source_root / source_metadata_jsonl
101
+ if not metadata_path.exists():
102
+ raise FileNotFoundError(f"metadata_jsonl does not exist: {metadata_path}")
103
+
104
+ motion_chunk_frames = action_chunk_frames + 1
105
+ chunk_duration_sec = float(action_chunk_frames) / motion_fps
106
+ output_metadata_path = output_root / shard_metadata_jsonl
107
+ final_metadata_path = output_root / "metadata.jsonl"
108
+
109
+
110
+ def stable_hash_mod(value: str, modulo: int) -> int:
111
+ digest = hashlib.sha1(value.encode("utf-8")).digest()
112
+ return int.from_bytes(digest[:8], "big") % int(modulo)
113
+
114
+
115
+ def shard_key(row: dict) -> str:
116
+ return str(row.get("name") or row.get("id") or row.get("video_path") or row.get("original_video_path") or "")
117
+
118
+
119
+ def parse_duration(row: dict) -> float:
120
+ for key in ("duration", "video_duration_sec"):
121
+ value = row.get(key)
122
+ if isinstance(value, (int, float)) and float(value) > 0:
123
+ return float(value)
124
+ meta_info = row.get("meta_info")
125
+ if isinstance(meta_info, dict):
126
+ value = meta_info.get("meta_duration")
127
+ if isinstance(value, (int, float)) and float(value) > 0:
128
+ return float(value)
129
+ return 0.0
130
+
131
+
132
+ def probe_duration(video_path: Path) -> float:
133
+ cmd = [
134
+ ffprobe_bin,
135
+ "-v",
136
+ "error",
137
+ "-select_streams",
138
+ "v:0",
139
+ "-show_entries",
140
+ "stream=duration:format=duration",
141
+ "-of",
142
+ "json",
143
+ str(video_path),
144
+ ]
145
+ proc = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, check=False)
146
+ if proc.returncode != 0:
147
+ raise RuntimeError(proc.stderr.decode("utf-8", errors="ignore").strip())
148
+ info = json.loads(proc.stdout.decode("utf-8"))
149
+ streams = info.get("streams") or []
150
+ duration = 0.0
151
+ if streams:
152
+ duration = float(streams[0].get("duration") or 0.0)
153
+ if duration <= 0:
154
+ duration = float(info.get("format", {}).get("duration") or 0.0)
155
+ if duration <= 0:
156
+ raise RuntimeError(f"bad duration for {video_path}")
157
+ return duration
158
+
159
+
160
+ def caption_for_row(row: dict) -> str:
161
+ for key in ("video_caption", "caption", "instruction", "text"):
162
+ value = row.get(key)
163
+ if isinstance(value, str) and value.strip():
164
+ return value.strip()
165
+ return default_caption
166
+
167
+
168
+ rows = []
169
+ with metadata_path.open("r", encoding="utf-8") as f:
170
+ for line in f:
171
+ line = line.strip()
172
+ if line:
173
+ rows.append(json.loads(line))
174
+
175
+ if limit > 0:
176
+ rows = rows[:limit]
177
+
178
+ selected_rows = [row for row in rows if shard_num <= 1 or stable_hash_mod(shard_key(row), shard_num) == shard_id]
179
+ print(f"[SHARD] build metadata shard={shard_id}/{shard_num} sources={len(selected_rows)}/{len(rows)}")
180
+
181
+ sources = []
182
+ missing_videos = 0
183
+ for row in selected_rows:
184
+ video_value = row.get("video_path") or row.get("original_video") or row.get("real_path")
185
+ if not video_value:
186
+ continue
187
+ source_video_path = source_root / str(video_value)
188
+ if not source_video_path.exists():
189
+ missing_videos += 1
190
+ continue
191
+ base_name = str(row.get("name") or row.get("id") or source_video_path.stem)
192
+ source_id = str(row.get("id") or base_name)
193
+ duration = parse_duration(row)
194
+ sources.append(
195
+ {
196
+ "base_name": base_name,
197
+ "source_id": source_id,
198
+ "source_video_path": source_video_path.resolve(),
199
+ "source_video_duration_sec": duration,
200
+ "video_caption": caption_for_row(row),
201
+ "motion_type": str(row.get("motion_type") or row.get("source") or dataset_name),
202
+ }
203
+ )
204
+
205
+ needs_probe = [idx for idx, source in enumerate(sources) if source["source_video_duration_sec"] <= 0]
206
+ if needs_probe:
207
+ max_workers = min(build_num_workers, 32)
208
+ with concurrent.futures.ThreadPoolExecutor(max_workers=max_workers) as executor:
209
+ future_map = {
210
+ executor.submit(probe_duration, sources[idx]["source_video_path"]): idx
211
+ for idx in needs_probe
212
+ }
213
+ for future in concurrent.futures.as_completed(future_map):
214
+ idx = future_map[future]
215
+ sources[idx]["source_video_duration_sec"] = future.result()
216
+
217
+ tmp_path = output_metadata_path.with_name(f".{output_metadata_path.name}.{os.getpid()}.{uuid.uuid4().hex}.tmp")
218
+ row_count = 0
219
+ skipped_short = 0
220
+ with tmp_path.open("w", encoding="utf-8") as out:
221
+ for source_idx, source in enumerate(sources, start=1):
222
+ duration_sec = float(source["source_video_duration_sec"])
223
+ max_start_frame = int((duration_sec - chunk_duration_sec) * motion_fps)
224
+ if max_start_frame < 0:
225
+ skipped_short += 1
226
+ continue
227
+ chunk_idx = 0
228
+ for chunk_start_frame in range(0, max_start_frame + 1, chunk_stride_frames):
229
+ chunk_name = f"{source['base_name']}.chunk{chunk_idx:02d}.f{chunk_start_frame:06d}"
230
+ row = {
231
+ "id": f"{source['source_id']}.chunk{chunk_idx:02d}.f{chunk_start_frame:06d}",
232
+ "name": chunk_name,
233
+ "converted_path": f"motion/{chunk_name}",
234
+ "video_path": f"videos/{chunk_name}.mp4",
235
+ "video_caption": source["video_caption"],
236
+ "duration": chunk_duration_sec,
237
+ "motion_fps": motion_fps,
238
+ "clip_frames": motion_chunk_frames,
239
+ "context_frames": context_frames,
240
+ "action_chunk_frames": action_chunk_frames,
241
+ "motion_chunk_frames": motion_chunk_frames,
242
+ "chunk_stride_frames": chunk_stride_frames,
243
+ "clip_selection_mode": clip_selection_mode,
244
+ "random_seed": clip_random_seed,
245
+ "middle_fraction": clip_middle_fraction,
246
+ "context_start_frame": chunk_start_frame,
247
+ "chunk_start_frame": chunk_start_frame,
248
+ "chunk_start_sec": float(chunk_start_frame) / motion_fps,
249
+ "source_video_duration_sec": duration_sec,
250
+ "source_video_path": str(source["source_video_path"]),
251
+ "motion_type": source["motion_type"],
252
+ "metadata_only": True,
253
+ }
254
+ out.write(json.dumps(row, ensure_ascii=False) + "\n")
255
+ row_count += 1
256
+ chunk_idx += 1
257
+ if chunk_progress_interval > 0 and source_idx % chunk_progress_interval == 0:
258
+ print(
259
+ f"[BUILD] dataset={dataset_name} shard={shard_id}/{shard_num} "
260
+ f"sources={source_idx}/{len(sources)} rows={row_count}"
261
+ )
262
+
263
+ tmp_path.replace(output_metadata_path)
264
+ print(
265
+ f"[DONE] dataset={dataset_name} metadata_rows={row_count} "
266
+ f"skipped_short={skipped_short} missing_videos={missing_videos} "
267
+ f"output={output_metadata_path}"
268
+ )
269
+
270
+ if shard_num > 1:
271
+ shard_paths = [output_root / f"metadata.shard-{idx:05d}-of-{shard_num:05d}.jsonl" for idx in range(shard_num)]
272
+ if all(path.exists() for path in shard_paths):
273
+ merge_tmp = final_metadata_path.with_name(f".{final_metadata_path.name}.{os.getpid()}.{uuid.uuid4().hex}.tmp")
274
+ with merge_tmp.open("w", encoding="utf-8") as out:
275
+ for path in shard_paths:
276
+ with path.open("r", encoding="utf-8") as src:
277
+ for line in src:
278
+ out.write(line)
279
+ merge_tmp.replace(final_metadata_path)
280
+ print(f"[MERGE] dataset={dataset_name} output={final_metadata_path}")
281
+ PY
282
+ }
283
+
284
+ parse_dataset_names() {
285
+ local raw_names="${1:-all}"
286
+ local token=""
287
+ DATASET_NAME_LIST=()
288
+ raw_names="${raw_names//,/ }"
289
+ for token in $raw_names; do
290
+ if [ "$token" = "all" ]; then
291
+ DATASET_NAME_LIST=(all)
292
+ return
293
+ fi
294
+ [ -n "$token" ] && DATASET_NAME_LIST+=("$token")
295
+ done
296
+ if [ "${#DATASET_NAME_LIST[@]}" -eq 0 ]; then
297
+ DATASET_NAME_LIST=(all)
298
+ fi
299
+ }
300
+
301
+ is_known_dataset() {
302
+ local candidate="$1"
303
+ local known=""
304
+ for known in "${KNOWN_DATASETS[@]}"; do
305
+ if [ "$known" = "$candidate" ]; then
306
+ return 0
307
+ fi
308
+ done
309
+ return 1
310
+ }
311
+
312
+ validate_dataset_names() {
313
+ local invalid=()
314
+ local name=""
315
+ parse_dataset_names "$DATASET_NAMES"
316
+ if [ "${#DATASET_NAME_LIST[@]}" -eq 1 ] && [ "${DATASET_NAME_LIST[0]}" = "all" ]; then
317
+ return
318
+ fi
319
+ for name in "${DATASET_NAME_LIST[@]}"; do
320
+ if ! is_known_dataset "$name"; then
321
+ invalid+=("$name")
322
+ fi
323
+ done
324
+ if [ "${#invalid[@]}" -gt 0 ]; then
325
+ echo "Unknown dataset name(s): ${invalid[*]}" >&2
326
+ echo "Available dataset names: ${KNOWN_DATASETS[*]}" >&2
327
+ exit 2
328
+ fi
329
+ }
330
+
331
+ dataset_selected() {
332
+ local dataset_name="$1"
333
+ local selected=""
334
+ if [ "${#DATASET_NAME_LIST[@]}" -eq 1 ] && [ "${DATASET_NAME_LIST[0]}" = "all" ]; then
335
+ return 0
336
+ fi
337
+ for selected in "${DATASET_NAME_LIST[@]}"; do
338
+ if [ "$selected" = "$dataset_name" ]; then
339
+ return 0
340
+ fi
341
+ done
342
+ return 1
343
+ }
344
+
345
+ run_video_only_cache() {
346
+ local output_root="$1"
347
+ local embed_root="$2"
348
+ local dataset_name="$3"
349
+ local metadata_jsonl
350
+ metadata_jsonl=$(metadata_jsonl_for_current_shard)
351
+
352
+ local shard_args=()
353
+ local limit_args=()
354
+ local embed_overwrite_args=()
355
+ local embed_compress_args=()
356
+
357
+ if (( SHARD_NUM_TOTAL > 1 )); then
358
+ shard_args+=(--shard-num "$SHARD_NUM_TOTAL" --shard-id "$SHARD_ID_VALUE")
359
+ fi
360
+ if [ "${LIMIT:-0}" -gt 0 ]; then
361
+ limit_args+=(--limit "${LIMIT}")
362
+ fi
363
+ if [ "${OVERWRITE_EMBEDS:-0}" = "1" ]; then
364
+ embed_overwrite_args+=(--overwrite)
365
+ fi
366
+ if [ "${EMBED_SAVE_COMPRESSED:-0}" = "1" ]; then
367
+ embed_compress_args+=(--save-compressed)
368
+ fi
369
+
370
+ mkdir -p "$output_root"
371
+ echo "------------------------------------------"
372
+ echo "Caching extra video-only dataset: ${dataset_name}"
373
+ echo "Dataset root: ${output_root}"
374
+ echo "Embed root: ${embed_root}"
375
+ echo "Metadata: ${metadata_jsonl}"
376
+ echo "Shard: ${SHARD_ID_VALUE}/${SHARD_NUM_TOTAL}"
377
+ echo "------------------------------------------"
378
+
379
+ "$PYTHON_BIN" "$SCRIPT_ROOT/cache_video_omni_embeds.py" \
380
+ --dataset-root "$output_root" \
381
+ --metadata-jsonl "$metadata_jsonl" \
382
+ --embed-root "$embed_root" \
383
+ --model-path "$MODEL_PATH" \
384
+ --video-sample-fps "$VIDEO_SAMPLE_FPS" \
385
+ --video-size-pixels "$VIDEO_SIZE_PIXELS" \
386
+ --video-decoder "$VIDEO_DECODER" \
387
+ --dtype float16 \
388
+ --num-workers "$EMBED_NUM_WORKERS" \
389
+ --existing-cache-check "$EXISTING_CACHE_CHECK" \
390
+ "${shard_args[@]}" \
391
+ --input-already-sharded \
392
+ --direct-source-clips \
393
+ "${limit_args[@]}" \
394
+ "${embed_compress_args[@]}" \
395
+ "${embed_overwrite_args[@]}"
396
+ }
397
+
398
+ validate_dataset_names
399
+
400
+ if [ "$#" -ne 0 ]; then
401
+ echo "This script does not take positional arguments. Use SHARD_ID=<id> instead." >&2
402
+ exit 2
403
+ fi
404
+
405
+ SHARD_ID_VALUE="${SHARD_ID:-}"
406
+ if [ -z "$SHARD_ID_VALUE" ]; then
407
+ echo "Please set SHARD_ID in [0, $((SHARD_NUM_TOTAL - 1))]. Example: SHARD_ID=0 DATASET_NAMES=2264_with_audio bash $0" >&2
408
+ exit 2
409
+ fi
410
+ if ! [[ "$SHARD_ID_VALUE" =~ ^[0-9]+$ ]]; then
411
+ echo "Shard id must be an integer in [0, $((SHARD_NUM_TOTAL - 1))]" >&2
412
+ exit 2
413
+ fi
414
+ if [ "$SHARD_ID_VALUE" -ge "$SHARD_NUM_TOTAL" ]; then
415
+ echo "Shard id must be an integer in [0, $((SHARD_NUM_TOTAL - 1))]" >&2
416
+ exit 2
417
+ fi
418
+
419
+ export PSI_VIDEO_CLIP_DATASET_ROOT=$DATASET_ROOT
420
+ export PSI_VIDEO_DEMO_ROOT=${PSI_VIDEO_DEMO_ROOT:-/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/videogen/motion_example_0506_demo}
421
+ EXTRA_VIDEO_2264_ROOT=${PSI_VIDEO_2264_ROOT:-/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/videogen/2264_with_audio}
422
+ EXTRA_VIDEO_2264_OUTPUT_ROOT=${PSI_VIDEO_2264_OUTPUT_ROOT:-$DATASET_ROOT/2264_with_audio}
423
+ EXTRA_VIDEO_2264_EMBED_ROOT=${PSI_VIDEO_2264_EMBED_ROOT:-$EXTRA_VIDEO_2264_OUTPUT_ROOT/embeds}
424
+ EXTRA_VIDEO_DISTILLED_ROOT=${PSI_VIDEO_DISTILLED_ROOT:-/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/videogen/distilled_video_0226}
425
+ EXTRA_VIDEO_DISTILLED_OUTPUT_ROOT=${PSI_VIDEO_DISTILLED_OUTPUT_ROOT:-$DATASET_ROOT/distilled_video_0226}
426
+ EXTRA_VIDEO_DISTILLED_EMBED_ROOT=${PSI_VIDEO_DISTILLED_EMBED_ROOT:-$EXTRA_VIDEO_DISTILLED_OUTPUT_ROOT/embeds}
427
+
428
+ export BUILD_CLIPS=${BUILD_CLIPS:-1}
429
+ export CACHE_EMBEDS=${CACHE_EMBEDS:-1}
430
+ export BUILD_DANCE=${BUILD_DANCE:-1}
431
+ export BUILD_MOTION_EXAMPLE=${BUILD_MOTION_EXAMPLE:-1}
432
+ export CACHE_DANCE=${CACHE_DANCE:-1}
433
+ export CACHE_MOTION_EXAMPLE=${CACHE_MOTION_EXAMPLE:-1}
434
+ export CACHE_2264_WITH_AUDIO=${CACHE_2264_WITH_AUDIO:-1}
435
+ export CACHE_DISTILLED_VIDEO_0226=${CACHE_DISTILLED_VIDEO_0226:-1}
436
+
437
+ if ! dataset_selected "dance_1723_video_10s"; then
438
+ export BUILD_DANCE=0
439
+ export CACHE_DANCE=0
440
+ fi
441
+
442
+ if ! dataset_selected "motion_example_0506_demo"; then
443
+ export BUILD_MOTION_EXAMPLE=0
444
+ export CACHE_MOTION_EXAMPLE=0
445
+ fi
446
+
447
+ if ! dataset_selected "2264_with_audio"; then
448
+ export CACHE_2264_WITH_AUDIO=0
449
+ fi
450
+
451
+ if ! dataset_selected "distilled_video_0226"; then
452
+ export CACHE_DISTILLED_VIDEO_0226=0
453
+ fi
454
+
455
+ export METADATA_ONLY=1
456
+ export DIRECT_SOURCE_CLIPS=1
457
+ export CLIP_SELECTION_MODE=all
458
+ export CHUNK_STRIDE_FRAMES=1
459
+ export ACTION_CHUNK_FRAMES=30
460
+ export MOTION_FPS=50
461
+ export CLIPS_PER_VIDEO=1
462
+ export CHUNK_PROGRESS_INTERVAL=${CHUNK_PROGRESS_INTERVAL:-2000}
463
+
464
+ export BUILD_NUM_WORKERS=${BUILD_NUM_WORKERS:-64}
465
+ export CHUNK_NUM_WORKERS=${CHUNK_NUM_WORKERS:-1}
466
+ # export EMBED_NUM_WORKERS=${EMBED_NUM_WORKERS:-64}
467
+ export EMBED_NUM_WORKERS=${EMBED_NUM_WORKERS:-32}
468
+ export EMBED_SAVE_COMPRESSED=${EMBED_SAVE_COMPRESSED:-0}
469
+ export EXISTING_CACHE_CHECK=${EXISTING_CACHE_CHECK:-size}
470
+ export CLIP_FRAMES=${CLIP_FRAMES:-50}
471
+ export CLIP_RANDOM_SEED=${CLIP_RANDOM_SEED:-42}
472
+ export CLIP_MIDDLE_FRACTION=${CLIP_MIDDLE_FRACTION:-0.8}
473
+ export VIDEO_SAMPLE_FPS=${VIDEO_SAMPLE_FPS:-8}
474
+ export VIDEO_SIZE_PIXELS=${VIDEO_SIZE_PIXELS:-50176}
475
+ export VIDEO_DECODER=${VIDEO_DECODER:-ffmpeg}
476
+ export SHARD_NUM=$SHARD_NUM_TOTAL
477
+ export SHARD_ID=$SHARD_ID_VALUE
478
+ export OVERWRITE_CLIPS=${OVERWRITE_CLIPS:-0}
479
+ export OVERWRITE_EMBEDS=${OVERWRITE_EMBEDS:-0}
480
+ export OMP_NUM_THREADS=${OMP_NUM_THREADS:-1}
481
+ MODEL_PATH=${PSI_PRETRAIN_BACKBONE_PATH:-/cpfs01/VideoGen/public_ckpt/Qwen/Qwen2.5-Omni-3B}
482
+
483
+ echo "=========================================="
484
+ echo "Running direct dense video_clip shard ${SHARD_ID_VALUE}/${SHARD_NUM_TOTAL}"
485
+ echo "Output root: ${DATASET_ROOT}"
486
+ echo "Datasets: ${DATASET_NAMES}"
487
+ echo "=========================================="
488
+
489
+ if [ "$BUILD_DANCE" = "1" ] || [ "$BUILD_MOTION_EXAMPLE" = "1" ] || [ "$CACHE_DANCE" = "1" ] || [ "$CACHE_MOTION_EXAMPLE" = "1" ]; then
490
+ bash cache_video_omni_embeds.sh
491
+ fi
492
+
493
+ if [ "$BUILD_CLIPS" = "1" ] || [ "$CACHE_EMBEDS" = "1" ]; then
494
+ prepare_python_env
495
+ fi
496
+
497
+ if [ "$BUILD_CLIPS" = "1" ] && [ "$CACHE_2264_WITH_AUDIO" = "1" ]; then
498
+ build_video_only_dense_metadata \
499
+ "$EXTRA_VIDEO_2264_ROOT" \
500
+ "$EXTRA_VIDEO_2264_OUTPUT_ROOT" \
501
+ "2264_with_audio"
502
+ fi
503
+
504
+ if [ "$CACHE_EMBEDS" = "1" ] && [ "$CACHE_2264_WITH_AUDIO" = "1" ]; then
505
+ run_video_only_cache \
506
+ "$EXTRA_VIDEO_2264_OUTPUT_ROOT" \
507
+ "$EXTRA_VIDEO_2264_EMBED_ROOT" \
508
+ "2264_with_audio"
509
+ fi
510
+
511
+ if [ "$BUILD_CLIPS" = "1" ] && [ "$CACHE_DISTILLED_VIDEO_0226" = "1" ]; then
512
+ build_video_only_dense_metadata \
513
+ "$EXTRA_VIDEO_DISTILLED_ROOT" \
514
+ "$EXTRA_VIDEO_DISTILLED_OUTPUT_ROOT" \
515
+ "distilled_video_0226"
516
+ fi
517
+
518
+ if [ "$CACHE_EMBEDS" = "1" ] && [ "$CACHE_DISTILLED_VIDEO_0226" = "1" ]; then
519
+ run_video_only_cache \
520
+ "$EXTRA_VIDEO_DISTILLED_OUTPUT_ROOT" \
521
+ "$EXTRA_VIDEO_DISTILLED_EMBED_ROOT" \
522
+ "distilled_video_0226"
523
+ fi
cache_video_omni_embeds_dense_stride1_full.sh ADDED
@@ -0,0 +1,63 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/usr/bin/env bash
2
+ set -euo pipefail
3
+
4
+ cd /cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/Psi0-yth-v18
5
+
6
+ DATASET_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/videogen/psi0_video_clip_50f_dense_stride1_full
7
+ SHARD_NUM_TOTAL=4
8
+
9
+ if [ "$#" -eq 0 ]; then
10
+ for shard_id in 0 1; do
11
+ echo "=========================================="
12
+ echo "Running dense video_clip shard ${shard_id}/${SHARD_NUM_TOTAL}"
13
+ echo "Output root: ${DATASET_ROOT}"
14
+ echo "=========================================="
15
+ bash "$0" "$shard_id"
16
+ done
17
+ exit 0
18
+ fi
19
+
20
+ SHARD_ID_VALUE="$1"
21
+ if ! [[ "$SHARD_ID_VALUE" =~ ^[0-9]+$ ]]; then
22
+ echo "Shard id must be one of: 0 1" >&2
23
+ exit 2
24
+ fi
25
+ if [ "$SHARD_ID_VALUE" -ge "$SHARD_NUM_TOTAL" ]; then
26
+ echo "Shard id must be one of: 0 1" >&2
27
+ exit 2
28
+ fi
29
+
30
+ export PSI_VIDEO_CLIP_DATASET_ROOT=$DATASET_ROOT
31
+ export PSI_VIDEO_DEMO_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/videogen/motion_example_0506_demo
32
+
33
+ export BUILD_CLIPS=1
34
+ export CACHE_EMBEDS=1
35
+ export BUILD_DANCE=1
36
+ export BUILD_MOTION_EXAMPLE=1
37
+ export CACHE_DANCE=1
38
+ export CACHE_MOTION_EXAMPLE=1
39
+
40
+ export CLIP_SELECTION_MODE=all
41
+ export CHUNK_STRIDE_FRAMES=1
42
+ export ACTION_CHUNK_FRAMES=30
43
+ export MOTION_FPS=50
44
+ export CLIPS_PER_VIDEO=1
45
+ export CHUNK_PROGRESS_INTERVAL=200
46
+
47
+ export BUILD_NUM_WORKERS=64
48
+ export CHUNK_NUM_WORKERS=2
49
+ export FFMPEG_THREADS=2
50
+ export X264_PRESET=veryfast
51
+ export EXISTING_VIDEO_CHECK=size
52
+ export EMBED_NUM_WORKERS=64
53
+ export VIDEO_SAMPLE_FPS=8
54
+ export VIDEO_SIZE_PIXELS=50176
55
+ export VIDEO_DECODER=ffmpeg
56
+ export SHARD_NUM=$SHARD_NUM_TOTAL
57
+ export SHARD_ID=$SHARD_ID_VALUE
58
+ export OVERWRITE_CLIPS=0
59
+ export OVERWRITE_EMBEDS=0
60
+ export OMP_NUM_THREADS=1
61
+
62
+ exec bash cache_video_omni_embeds.sh
63
+ sleep 86400
cache_video_omni_embeds_full.sh ADDED
@@ -0,0 +1,31 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/usr/bin/env bash
2
+ set -euo pipefail
3
+
4
+ cd /cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/Psi0-yth-v18
5
+
6
+ export PSI_VIDEO_CLIP_DATASET_ROOT=${PSI_VIDEO_CLIP_DATASET_ROOT:-/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/videogen/psi0_video_clip_50f_rand32_full}
7
+ export PSI_VIDEO_DEMO_ROOT=${PSI_VIDEO_DEMO_ROOT:-/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/videogen/motion_example_0506_demo}
8
+
9
+ export BUILD_CLIPS=${BUILD_CLIPS:-1}
10
+ export CACHE_EMBEDS=${CACHE_EMBEDS:-1}
11
+ export BUILD_DANCE=${BUILD_DANCE:-1}
12
+ export BUILD_MOTION_EXAMPLE=${BUILD_MOTION_EXAMPLE:-1}
13
+ export CACHE_DANCE=${CACHE_DANCE:-1}
14
+ export CACHE_MOTION_EXAMPLE=${CACHE_MOTION_EXAMPLE:-1}
15
+
16
+ export CLIP_SELECTION_MODE=all
17
+ export CHUNK_STRIDE_FRAMES=${CHUNK_STRIDE_FRAMES:-20}
18
+ export ACTION_CHUNK_FRAMES=${ACTION_CHUNK_FRAMES:-30}
19
+ export MOTION_FPS=${MOTION_FPS:-50}
20
+ export CLIPS_PER_VIDEO=${CLIPS_PER_VIDEO:-1}
21
+
22
+ export BUILD_NUM_WORKERS=${BUILD_NUM_WORKERS:-256}
23
+ export EMBED_NUM_WORKERS=${EMBED_NUM_WORKERS:-256}
24
+ export VIDEO_SAMPLE_FPS=${VIDEO_SAMPLE_FPS:-32}
25
+ export SHARD_NUM=${SHARD_NUM:-4}
26
+ export SHARD_ID=${SHARD_ID:-0}
27
+ export OVERWRITE_CLIPS=${OVERWRITE_CLIPS:-1}
28
+ export OVERWRITE_EMBEDS=${OVERWRITE_EMBEDS:-1}
29
+ export OMP_NUM_THREADS=${OMP_NUM_THREADS:-1}
30
+
31
+ exec bash cache_video_omni_embeds.sh
dataset/seed/g1/csv/220714/jog_backward_start_001__A025.csv ADDED
The diff for this file is too large to render. See raw diff
 
dataset/seed/g1/csv/220714/jog_backward_start_003__A025.csv ADDED
The diff for this file is too large to render. See raw diff
 
dataset/seed/g1/csv/220714/walk_forward_stop_003__A025.csv ADDED
The diff for this file is too large to render. See raw diff
 
dataset/seed/g1/csv/230215/idle_crouch_right_loop_R_003__A194.csv ADDED
The diff for this file is too large to render. See raw diff
 
dataset/seed/g1/csv/230306/crouch_idle_004__A244.csv ADDED
The diff for this file is too large to render. See raw diff
 
dataset/seed/g1/csv/240327/jump_twice_R_001__A533.csv ADDED
The diff for this file is too large to render. See raw diff
 
dataset/seed/g1/csv/240327/jump_twice_R_002__A533.csv ADDED
The diff for this file is too large to render. See raw diff
 
dataset/seed/g1/csv/240327/jump_twice_R_003__A533.csv ADDED
The diff for this file is too large to render. See raw diff
 
dataset/seed/g1/csv/240327/touch_toes_standing_R_001__A533.csv ADDED
The diff for this file is too large to render. See raw diff
 
dataset/seed/metadata/seed_metadata_v004_100.csv ADDED
@@ -0,0 +1,101 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ move_name,filename,move_duration_frames,package,category,is_neutral,is_mirror,move_soma_uniform_path,move_soma_uniform_shape_path,move_soma_proportional_path,move_soma_proportional_shape_path,move_g1_path,take_name,take_actor,take_org_name,take_date,take_day_part,content_name,content_natural_desc_1,content_natural_desc_2,content_natural_desc_3,content_natural_desc_4,content_technical_description,content_short_description,content_short_description_2,content_all_rigplay_styles,content_uniform_style,content_type_of_movement,content_body_position,content_horizontal_move,content_vertical_move,content_props,content_complex_action,content_repeated_action,actor_uid,actor_height,actor_height_cm,actor_foot_cm,actor_collarbone_height_cm,actor_collarbone_span_cm,actor_elbow_span_cm,actor_wrist_span_cm,actor_shoulder_span_cm,actor_hips_height_cm,actor_hips_bones_span_cm,actor_knee_height_cm,actor_ankle_height_cm,actor_weight_kg,actor_age_yr,actor_gender,actor_profession
2
+ crouch_idle_001__A147,idle_crouch_loop_001__A147,761,Locomotion,Advanced Locomotion,1.0,False,soma_uniform/bvh/230125/idle_crouch_loop_001__A147.bvh,soma_shapes/soma_base_fit_mhr_params.npz,soma_proportional/bvh/230125/idle_crouch_loop_001__A147.bvh,soma_shapes/soma_proportion_fit_mhr_params/A147.npz,g1/csv/230125/idle_crouch_loop_001__A147.csv,crouch_idle_001,A147,idle_crouch_loop_001,230125,_2,crouch_idle,assume a crouching position with feet almost parallel,assumes a squatting posture with nearly parallel feet in an open position,person is crouching down with their feet in a parallel and open position,A person is sitting in a crouched posture.,"feet almost parallel in open position, talent crouching",crouch position,crouch idle,neutral,neutral,walking,crouching,0,0,0,0,0,A147,M,183,28,150,30,92,143,182,100,31,57,10,84,26,M,actor
3
+ high_jump_R_opt_1_001__A280,high_jump_R_001__A280,639,Locomotion,Baseline,1.0,False,soma_uniform/bvh/230316/high_jump_R_001__A280.bvh,soma_shapes/soma_base_fit_mhr_params.npz,soma_proportional/bvh/230316/high_jump_R_001__A280.bvh,soma_shapes/soma_proportion_fit_mhr_params/A280.npz,g1/csv/230316/high_jump_R_001__A280.csv,high_jump_R_opt_1_001,A280,high_jump_R_001,230316,_1,high_jump_R_opt_1,high jump over a moving pole in one place,making high jump over a moving pole stationary,performer makes high jump in one place over a moving pole,A person energetically does a high jump while facing forward.,"jump high over moving pole, jump high in one place",jumping high,high jump opt 1,neutral,neutral,jumping,standing,1,1,0,0,0,A280,M,171,27,141,32,93,134,170,99,27,50,9,68,25,M,actor
4
+ crouch_idle_right_R_001__A194,idle_crouch_right_loop_R_001__A194,1380,Locomotion,Advanced Locomotion,1.0,False,soma_uniform/bvh/230215/idle_crouch_right_loop_R_001__A194.bvh,soma_shapes/soma_base_fit_mhr_params.npz,soma_proportional/bvh/230215/idle_crouch_right_loop_R_001__A194.bvh,soma_shapes/soma_proportion_fit_mhr_params/A194.npz,g1/csv/230215/idle_crouch_right_loop_R_001__A194.csv,crouch_idle_right_R_001,A194,idle_crouch_right_loop_R_001,230215,_1,crouch_idle_right_R,character with right foot forward assumes a crouching position leaning on the left foot,stand in a crouching position with right foot forward,maintain a crouching position with the right leg slightly extended forward,A crouched person is standing in place with their right leg ahead of the left.,"right foot slightly forward, body weight on left foot, talent crouching",crouch position right leg forward,crouch idle right,neutral,neutral,crouching idle,crouching,0,0,0,0,0,A194,M,178,29,146,29,91,138,173,98,32,53,10,90,25,M,actor
5
+ walk_ff_start_315_R_001__A049,walk_ff_start_315_001__A049,692,Locomotion,Basic Locomotion Neutral,1.0,False,soma_uniform/bvh/221011/walk_ff_start_315_001__A049.bvh,soma_shapes/soma_base_fit_mhr_params.npz,soma_proportional/bvh/221011/walk_ff_start_315_001__A049.bvh,soma_shapes/soma_proportion_fit_mhr_params/A049.npz,g1/csv/221011/walk_ff_start_315_001__A049.csv,walk_ff_start_315_R_001,A049,walk_ff_start_315_001,221011,_1,walk_ff_start_315_R,walking backward starting with right foot and moving diagonally with left foot crossing behind,"begin by moving right foot, then move backward diagonally crossing with left foot",person takes a right diagonal step backward,A person starts walking swiftly toward the back right diagonal in a crosswise manner at a moderate pace.,"start right foot, move backward diagonally, left foot back crossing, walk backward diagonally",start walking back right diagonal,walk ff start 315,neutral,neutral,walking,standing,1,0,0,0,0,A049,S,156,23,130,40,84,120,154,88,24,47,9,50,25,F,general
6
+ jog_arc_ccw_stop_001__A029,arc_jog_left_stop_001__A029,739,Locomotion,Basic Locomotion Neutral,1.0,False,soma_uniform/bvh/220720/arc_jog_left_stop_001__A029.bvh,soma_shapes/soma_base_fit_mhr_params.npz,soma_proportional/bvh/220720/arc_jog_left_stop_001__A029.bvh,soma_shapes/soma_proportion_fit_mhr_params/A029.npz,g1/csv/220720/arc_jog_left_stop_001__A029.csv,jog_arc_ccw_stop_001,A029,arc_jog_left_stop_001,220720,_1,jog_arc_ccw_stop,"person jogs in a counterclockwise circle, then abruptly stops",person stops jogging in a counterclockwise arc,stop jogging in a counterclockwise circle,A person jogs actively towards left in a semicircle.,"jogging in a circle, counterclockwise, stop",stop jogging counterclockwise in an arc,jog arc ccw stop,neutral,neutral,jogging,standing,1,1,0,0,0,A029,S,155,24,127,31,85,120,158,80,22,43,8,41,22,F,general
7
+ walk_ff_stop_360_R_001__A017,Stop_backward_Walk_001__A017,7854,Locomotion,Basic Locomotion Neutral,1.0,False,soma_uniform/bvh/220705/Stop_backward_Walk_001__A017.bvh,soma_shapes/soma_base_fit_mhr_params.npz,soma_proportional/bvh/220705/Stop_backward_Walk_001__A017.bvh,soma_shapes/soma_proportion_fit_mhr_params/A017.npz,g1/csv/220705/Stop_backward_Walk_001__A017.csv,walk_ff_stop_360_R_001,A017,Stop_backward_Walk_001,220705,_1,walk_ff_stop_360_R,person stops and walks backwards,stop the action of walking backwards,"individual ends their backward walk, starting with their left foot",The person stops walking backwards.,"end left foot, move backward, walk backward",stop walking backwards,walk ff stop 360,neutral,neutral,walking,standing,1,0,0,0,0,A017,T,180,27,148,43,95,144,178,100,27,50,9,60,32,F,general
8
+ crouch_ff_stop_180_R_001__A195,crouch_ff_stop_180_R_001__A195,1056,Locomotion,Advanced Locomotion,1.0,False,soma_uniform/bvh/230215/crouch_ff_stop_180_R_001__A195.bvh,soma_shapes/soma_base_fit_mhr_params.npz,soma_proportional/bvh/230215/crouch_ff_stop_180_R_001__A195.bvh,soma_shapes/soma_proportion_fit_mhr_params/A195.npz,g1/csv/230215/crouch_ff_stop_180_R_001__A195.csv,crouch_ff_stop_180_R_001,A195,crouch_ff_stop_180_R_001,230215,_2,crouch_ff_stop_180_R,walking in a forwards direction while in a crouching position,a crouching character stops and faces forward after moving forward,start walking forward while maintaining a crouch position,A person walking forward in a crouching position stops walking and stands in the same stance.,"end left foot, move forward, talent crouching, walking in crouch",stopping crouch facing forward,crouch ff stop 180,neutral,neutral,walking,crouching,1,1,0,0,0,A195,T,177,26,147,30,96,138,174,100,27,54,8,64,25,F,actor
9
+ turn_jump_000_001__A022,turn_jump_0000_001__A022,336,Locomotion,Basic Locomotion Neutral,1.0,False,soma_uniform/bvh/220713/turn_jump_0000_001__A022.bvh,soma_shapes/soma_base_fit_mhr_params.npz,soma_proportional/bvh/220713/turn_jump_0000_001__A022.bvh,soma_shapes/soma_proportion_fit_mhr_params/A022.npz,g1/csv/220713/turn_jump_0000_001__A022.csv,turn_jump_000_001,A022,turn_jump_0000_001,220713,_1,turn_jump_000,jumping back over the left shoulder with a full spin,a jump over the left shoulder with a 180-degree turn,person leaps one step while rotating 180 degrees over the left shoulder,A person quickly jumps back over their left shoulder and then stands.,rotate 180 degrees over your left shoulder while jumping one step,turn jump back over left shoulder,turn jump 000,neutral,neutral,"jumping, turning",standing,1,1,0,0,0,A022,T,180,25,152,32,93,135,174,105,28,51,7,62,30,F,general
10
+ jog_ff_start_180_R_001__A021,jog_forward_start_001__A021,675,Locomotion,Baseline,1.0,False,soma_uniform/bvh/220713/jog_forward_start_001__A021.bvh,soma_shapes/soma_base_fit_mhr_params.npz,soma_proportional/bvh/220713/jog_forward_start_001__A021.bvh,soma_shapes/soma_proportion_fit_mhr_params/A021.npz,g1/csv/220713/jog_forward_start_001__A021.csv,jog_ff_start_180_R_001,A021,jog_forward_start_001,220713,_1,jog_ff_start_180_R,start jogging forward,start a forward jog,initiate a forward jog,A person standing on the ground steadily jogs in the forward direction and then stands on the ground in an upright position.,"start right foot, move forward, jog",start jog forward,jog ff start 180,neutral,neutral,jogging,standing,1,1,0,0,0,A021,S,155,23,129,30,85,121,152,85,24,44,7,41,29,F,general
11
+ jog_arc_ccw_start_001__A029,arc_jog_left_start_001__A029,810,Locomotion,Basic Locomotion Neutral,1.0,False,soma_uniform/bvh/220720/arc_jog_left_start_001__A029.bvh,soma_shapes/soma_base_fit_mhr_params.npz,soma_proportional/bvh/220720/arc_jog_left_start_001__A029.bvh,soma_shapes/soma_proportion_fit_mhr_params/A029.npz,g1/csv/220720/arc_jog_left_start_001__A029.csv,jog_arc_ccw_start_001,A029,arc_jog_left_start_001,220720,_1,jog_arc_ccw_start,jogging is initiated in a counterclockwise arc,commence a jog in counterclockwise direction,person starts jogging in a counterclockwise arc,A person standing starts jogging in an anticlockwise arc.,"jogging in a circle, counterclockwise, start",start jogging counterclockwise in an arc,jog arc ccw start,neutral,neutral,jogging,standing,1,1,0,0,0,A029,S,155,24,127,31,85,120,158,80,22,43,8,41,22,F,general
12
+ jog_arc_cw_stop_R_001__A049,jog_arc_cw_stop_001__A049,727,Locomotion,Basic Locomotion Neutral,1.0,False,soma_uniform/bvh/221011/jog_arc_cw_stop_001__A049.bvh,soma_shapes/soma_base_fit_mhr_params.npz,soma_proportional/bvh/221011/jog_arc_cw_stop_001__A049.bvh,soma_shapes/soma_proportion_fit_mhr_params/A049.npz,g1/csv/221011/jog_arc_cw_stop_001__A049.csv,jog_arc_cw_stop_R_001,A049,jog_arc_cw_stop_001,221011,_1,jog_arc_cw_stop_R,"someone stops jogging in a clockwise, arc-like motion",stop and jog clockwise forming a circular path,person finishes jogging in a clockwise arc,"A person is jogging in a clockwise direction, forming an arc after which they stop at a pace.","end left foot, move forward in a circle, clockwise, jog ",stop jogging clockwise in an arc,jog arc cw stop,neutral,neutral,jogging,standing,1,1,0,0,0,A049,S,156,23,130,40,84,120,154,88,24,47,9,50,25,F,general
13
+ walk_ff_stop_180_R_001__A017,Stop_forward_Walk_001__A017,12547,Locomotion,Basic Locomotion Neutral,1.0,False,soma_uniform/bvh/220705/Stop_forward_Walk_001__A017.bvh,soma_shapes/soma_base_fit_mhr_params.npz,soma_proportional/bvh/220705/Stop_forward_Walk_001__A017.bvh,soma_shapes/soma_proportion_fit_mhr_params/A017.npz,g1/csv/220705/Stop_forward_Walk_001__A017.csv,walk_ff_stop_180_R_001,A017,Stop_forward_Walk_001,220705,_1,walk_ff_stop_180_R,stop moving forward by landing on the left foot while walking,"advance forward and stop with a flourish, landing on your left foot","finish walking forward, ending with the left foot",A person standing is walking in the forward direction at a steady pace.,"end left foot, move forward, walk forward, maximum expression","stop walking forward, finish with left leg",walk ff stop 180,neutral,neutral,walking,standing,1,0,0,0,0,A017,T,180,27,148,43,95,144,178,100,27,50,9,60,32,F,general
14
+ turn_walk_000_L_001__A021,turn_start_walk_0000_001__A021,982,Locomotion,Basic Locomotion Neutral,1.0,False,soma_uniform/bvh/220713/turn_start_walk_0000_001__A021.bvh,soma_shapes/soma_base_fit_mhr_params.npz,soma_proportional/bvh/220713/turn_start_walk_0000_001__A021.bvh,soma_shapes/soma_proportion_fit_mhr_params/A021.npz,g1/csv/220713/turn_start_walk_0000_001__A021.csv,turn_walk_000_L_001,A021,turn_start_walk_0000_001,220713,_1,turn_walk_000_L,figure turns 180 degrees over their left shoulder and then walks forward,a quick turn over the left shoulder followed by a forward walk,character turns 180 degrees over left shoulder and moves forward,The person turns back over the left shoulder and walks backward at a normal pace before stopping.,"start left foot, turn left 180 degrees over left shoulder, walk forward, move forward","turns back over left shoulder, walks forward",turn walk 000,neutral,neutral,"walking, turning",standing,1,0,0,0,0,A021,S,155,23,129,30,85,121,152,85,24,44,7,41,29,F,general
15
+ jog_ff_start_000_001__A025,jog_backward_start_001__A025,469,Locomotion,Basic Locomotion Neutral,1.0,False,soma_uniform/bvh/220714/jog_backward_start_001__A025.bvh,soma_shapes/soma_base_fit_mhr_params.npz,soma_proportional/bvh/220714/jog_backward_start_001__A025.bvh,soma_shapes/soma_proportion_fit_mhr_params/A025.npz,g1/csv/220714/jog_backward_start_001__A025.csv,jog_ff_start_000_001,A025,jog_backward_start_001,220714,_1,jog_ff_start_000,"start a backwards jog, leading with the left leg","jog backward, starting with the left leg","person performing a backwards jog, facing forward, initiating movement with the left leg",A standing person starts jogging backward steadily by taking medium-length strides while facing forward.,"face forward, jogging back, left leg first, start",begin jog backwards,jog ff start 000,neutral,neutral,jogging,standing,1,1,0,0,0,A025,T,199,31,165,44,103,157,201,113,28,58,10,102,19,M,general
16
+ jump_ff_000_001__A021,jump_backward_001__A021,665,Locomotion,Basic Locomotion Neutral,1.0,False,soma_uniform/bvh/220713/jump_backward_001__A021.bvh,soma_shapes/soma_base_fit_mhr_params.npz,soma_proportional/bvh/220713/jump_backward_001__A021.bvh,soma_shapes/soma_proportion_fit_mhr_params/A021.npz,g1/csv/220713/jump_backward_001__A021.csv,jump_ff_000_001,A021,jump_backward_001,220713,_1,jump_ff_000,"jumping back, landing on the right foot",jumping backwards and landing on the right leg,"a jump backwards led by the left foot, while constantly looking forwards",A person jumps backward and then stands in an upright posture.,"jumping back, left foot first, head looking forward","jump backwards, finish with right leg",jump ff 000,neutral,neutral,jumping,standing,1,1,0,0,0,A021,S,155,23,129,30,85,121,152,85,24,44,7,41,29,F,general
17
+ jog_ff_start_315_R_001__A050,jog_ff_start_315_001__A050,426,Locomotion,Basic Locomotion Neutral,1.0,False,soma_uniform/bvh/221011/jog_ff_start_315_001__A050.bvh,soma_shapes/soma_base_fit_mhr_params.npz,soma_proportional/bvh/221011/jog_ff_start_315_001__A050.bvh,soma_shapes/soma_proportion_fit_mhr_params/A050.npz,g1/csv/221011/jog_ff_start_315_001__A050.csv,jog_ff_start_315_R_001,A050,jog_ff_start_315_001,221011,_1,jog_ff_start_315_R,person starts jogging backward diagonally with the right foot,person jogs backward diagonally starting with the right foot,person starts a diagonal jog backward,A person swiftly jogs backward along the back-right diagonal.,"start right foot, move backward diagonally, left foot back crossing, jog",start jog back right diagonally,jog ff start 315,neutral,neutral,jogging,standing,1,1,0,0,0,A050,T,182,26,152,40,99,140,180,105,24,53,10,75,31,F,general
18
+ turn_jump_090_001__A023,turn_jump_0090_001__A023,600,Locomotion,Basic Locomotion Neutral,1.0,False,soma_uniform/bvh/220713/turn_jump_0090_001__A023.bvh,soma_shapes/soma_base_fit_mhr_params.npz,soma_proportional/bvh/220713/turn_jump_0090_001__A023.bvh,soma_shapes/soma_proportion_fit_mhr_params/A023.npz,g1/csv/220713/turn_jump_0090_001__A023.csv,turn_jump_090_001,A023,turn_jump_0090_001,220713,_1,turn_jump_090,individual performing a leftward turning jump step,a one-step jump with a 90-degree rotation over the left shoulder,individual turns 90 degrees over their left shoulder while jumping one step,A person swiftly pivots and jumps left with agility.,rotate 90 degrees over your left shoulder while jumping one step,turn jump left,turn jump 090,neutral,neutral,"jumping, turning",standing,1,1,0,0,0,A023,T,190,31,158,42,106,160,195,105,27,54,10,90,46,M,general
19
+ crouch_turn_360_R_001__A126,turn_crouch_360_001__A126,673,Locomotion,Advanced Locomotion,1.0,False,soma_uniform/bvh/230119/turn_crouch_360_001__A126.bvh,soma_shapes/soma_base_fit_mhr_params.npz,soma_proportional/bvh/230119/turn_crouch_360_001__A126.bvh,soma_shapes/soma_proportion_fit_mhr_params/A126.npz,g1/csv/230119/turn_crouch_360_001__A126.csv,crouch_turn_360_R_001,A126,turn_crouch_360_001,230119,_3,crouch_turn_360_R,character crouches and turns back over the right shoulder,character makes a 180 degree turn over the right shoulder while crouching,complete a turn back over the right shoulder while keeping a crouch,A person turns back over the right shoulder in a couching stance.,"start right foot, turn right 180 degrees over right shoulder, 3 steps, talent crouching, walking in crouch",turning back over right shoulder while crouching,crouch turn 360,neutral,neutral,turning,crouching,0,0,0,0,0,A126,M,169,25,136,25,90,130,164,95,31,51,8,63,25,F,actor
20
+ turn_walk_045_L_001__A023,turn_start_walk_0045_001__A023,538,Locomotion,Basic Locomotion Neutral,1.0,False,soma_uniform/bvh/220713/turn_start_walk_0045_001__A023.bvh,soma_shapes/soma_base_fit_mhr_params.npz,soma_proportional/bvh/220713/turn_start_walk_0045_001__A023.bvh,soma_shapes/soma_proportion_fit_mhr_params/A023.npz,g1/csv/220713/turn_start_walk_0045_001__A023.csv,turn_walk_045_L_001,A023,turn_start_walk_0045_001,220713,_1,turn_walk_045_L,character takes strides forward after a 135-degree left turn over the left shoulder,"character makes a 135 degree left turn, walking forward","person takes a left step, turns left 135 degrees and walks forward",A person turns around towards their left back diagonal and starts walking steadily.,"start left foot, turn left 135 degrees over left shoulder, walk forward diagonally",turns left walks forward,turn walk 045,neutral,neutral,"walking, turning",standing,1,0,0,0,0,A023,T,190,31,158,42,106,160,195,105,27,54,10,90,46,M,general
21
+ jump_twice_R_001__A533,jump_twice_R_001__A533,475,Locomotion,Basic Locomotion Neutral,1.0,False,soma_uniform/bvh/240327/jump_twice_R_001__A533.bvh,soma_shapes/soma_base_fit_mhr_params.npz,soma_proportional/bvh/240327/jump_twice_R_001__A533.bvh,soma_shapes/soma_proportion_fit_mhr_params/A533.npz,g1/csv/240327/jump_twice_R_001__A533.csv,jump_twice_R_001,A533,jump_twice_R_001,240327,_1,jump_twice_R,jump and jump again,person jumps forward twice,two big jumps one after the other,"A person steadily bends, jumps twice in the forward direction, then straightens up, and stands in an upright posture.",person jumps forward right leg first and then jumps again,jump twice in a row,jump twice,neutral,neutral,jumping,standing,1,1,0,0,1,A533,M,171,29,141,35,90,134,172,95,26,54,11,71,27,M,actor
22
+ relaxed_fast_walk_ff_180_001__A005,walk_fast_forward_relax_001__A005,2248,Locomotion,Basic Locomotion Styles,0.0,False,soma_uniform/bvh/210707/walk_fast_forward_relax_001__A005.bvh,soma_shapes/soma_base_fit_mhr_params.npz,soma_proportional/bvh/210707/walk_fast_forward_relax_001__A005.bvh,soma_shapes/soma_proportion_fit_mhr_params/A005.npz,g1/csv/210707/walk_fast_forward_relax_001__A005.csv,relaxed_fast_walk_ff_180_001,A005,walk_fast_forward_relax_001,210707,_1,relaxed_fast_walk_ff_180,a casual individual briskly walking forward with a sense of calmness,"person walks at a brisk pace forward, maintaining an open and relaxed posture","walk forward quickly, doing so with a relaxed, open posture","A person with arms raised sideways lowers them and walks forward quickly, turning around during the walk and then stands with their arms raised sideways.","person walks forward, high energy, open position, calm",relaxed fast walk facing forward,relaxed fast walk ff 180,relaxed,neutral,walking,standing,1,0,0,0,0,A005,S,152,25,129,32,85,120,153,92,28,45,7,50,23,F,general
23
+ crouch_ff_stop_360_R_001__A126,crouch_ff_stop_360_001__A126,847,Locomotion,Advanced Locomotion,1.0,False,soma_uniform/bvh/230119/crouch_ff_stop_360_001__A126.bvh,soma_shapes/soma_base_fit_mhr_params.npz,soma_proportional/bvh/230119/crouch_ff_stop_360_001__A126.bvh,soma_shapes/soma_proportion_fit_mhr_params/A126.npz,g1/csv/230119/crouch_ff_stop_360_001__A126.csv,crouch_ff_stop_360_R_001,A126,crouch_ff_stop_360_001,230119,_3,crouch_ff_stop_360_R,person stops crouched walk and turns over their right shoulder,"a crouching person, moves backward and stops over the right shoulder",person crouching walks backwards and stops over their right shoulder,The crouched person walks backward while facing forward with arms swaying.,"end left foot, move backward, talent crouching, walking in crouch",stopping crouch back over right shoulder,crouch ff stop 360,neutral,neutral,walking,crouching,1,1,0,0,0,A126,M,169,25,136,25,90,130,164,95,31,51,8,63,25,F,actor
24
+ high_jump_full_turn_R_opt_1_001__A479,high_jump_full_turn_R_opt_1_001__A479,303,Locomotion,Advanced Locomotion,1.0,False,soma_uniform/bvh/231011/high_jump_full_turn_R_opt_1_001__A479.bvh,soma_shapes/soma_base_fit_mhr_params.npz,soma_proportional/bvh/231011/high_jump_full_turn_R_opt_1_001__A479.bvh,soma_shapes/soma_proportion_fit_mhr_params/A479.npz,g1/csv/231011/high_jump_full_turn_R_opt_1_001__A479.csv,high_jump_full_turn_R_opt_1_001,A479,high_jump_full_turn_R_opt_1_001,231011,_1,high_jump_full_turn_R_opt_1,turning around during high jump up,the performer turns around while jumping high up,the performer turns around during a high jump,A person swiftly turns around while doing a high jump and then lands on the floor standing in a wide stance.,character turns around during a high jump up,turning around during a high jump up and landing,high jump full turn opt 1,neutral,neutral,jumping,standing,0,1,0,0,0,A479,M,183,31,149,32,102,147,187,103,29,55,10,84,27,M,actor
25
+ high_jump_R_opt_2_001__A476,high_jump_R_opt_2_001__A476,374,Locomotion,Basic Locomotion Neutral,1.0,False,soma_uniform/bvh/231011/high_jump_R_opt_2_001__A476.bvh,soma_shapes/soma_base_fit_mhr_params.npz,soma_proportional/bvh/231011/high_jump_R_opt_2_001__A476.bvh,soma_shapes/soma_proportion_fit_mhr_params/A476.npz,g1/csv/231011/high_jump_R_opt_2_001__A476.csv,high_jump_R_opt_2_001,A476,high_jump_R_opt_2_001,231011,_1,high_jump_R_opt_2,character jumps up high in one place,person performs a high jump in place,character jumps high in place,"A person standing upright jumps high in place and then stands in a place, swaying.",jumping high lifting first right then left leg,high jump in place,high jump opt 2,neutral,neutral,jumping,standing,1,1,0,0,0,A476,M,166,28,137,29,95,129,169,99,30,52,11,55,26,F,actor
26
+ jump_around_001__A492,jump_around_001__A492,1364,Locomotion,Advanced Locomotion,1.0,False,soma_uniform/bvh/231024/jump_around_001__A492.bvh,soma_shapes/soma_base_fit_mhr_params.npz,soma_proportional/bvh/231024/jump_around_001__A492.bvh,soma_shapes/soma_proportion_fit_mhr_params/A492.npz,g1/csv/231024/jump_around_001__A492.csv,jump_around_001,A492,jump_around_001,231024,_2,jump_around,jumping with hands in the air,jumping with hands high,jumping around with arms straight,A person is energetically jumping in place with their hands raised in the air.,character jumps with their hands in the air,jumping with hands raised in the air,jump around,neutral,neutral,jumping,"standing, leaning",0,1,0,0,0,A492,M,170,29,141,27,96,136,172,98,27,53,11,62,28,F,actor
27
+ high_jump_ff_180_R_opt_2_001__A476,high_jump_ff_180_R_opt_2_001__A476,497,Locomotion,Basic Locomotion Neutral,1.0,False,soma_uniform/bvh/231011/high_jump_ff_180_R_opt_2_001__A476.bvh,soma_shapes/soma_base_fit_mhr_params.npz,soma_proportional/bvh/231011/high_jump_ff_180_R_opt_2_001__A476.bvh,soma_shapes/soma_proportion_fit_mhr_params/A476.npz,g1/csv/231011/high_jump_ff_180_R_opt_2_001__A476.csv,high_jump_ff_180_R_opt_2_001,A476,high_jump_ff_180_R_opt_2_001,231011,_1,high_jump_ff_180_R_opt_2,character jumps high forward,person executes a dynamic forward jump,character executes forward jump,"A person is standing, then jumps high forward, lands flat, and again stands in a place while swaying their arms.",jumping high forward lifting first right then left leg,jumps high forward,high jump ff 180 opt 2,neutral,neutral,jumping,standing,1,1,0,0,0,A476,M,166,28,137,29,95,129,169,99,30,52,11,55,26,F,actor
28
+ high_jump_ff_180_R_opt_1_001__A476,high_jump_ff_180_R_opt_1_001__A476,413,Locomotion,Advanced Locomotion,1.0,False,soma_uniform/bvh/231011/high_jump_ff_180_R_opt_1_001__A476.bvh,soma_shapes/soma_base_fit_mhr_params.npz,soma_proportional/bvh/231011/high_jump_ff_180_R_opt_1_001__A476.bvh,soma_shapes/soma_proportion_fit_mhr_params/A476.npz,g1/csv/231011/high_jump_ff_180_R_opt_1_001__A476.csv,high_jump_ff_180_R_opt_1_001,A476,high_jump_ff_180_R_opt_1_001,231011,_1,high_jump_ff_180_R_opt_1,jumping high forward two legs at the same time,the performer jumps forward high with both legs,"jumping forward, the performer lifts both legs off the ground",A person standing upright jumps forward.,character jumps high forward with both legs at the same time,performing a high jump forward with both legs,high jump ff 180 opt 1,neutral,neutral,standing,standing,0,1,0,0,0,A476,M,166,28,137,29,95,129,169,99,30,52,11,55,26,F,actor
29
+ walk_ff_start_360_R_fast_001__A447,walk_ff_start_360_R_fast_001__A447,530,Locomotion,Basic Locomotion Neutral,1.0,False,soma_uniform/bvh/230906/walk_ff_start_360_R_fast_001__A447.bvh,soma_shapes/soma_base_fit_mhr_params.npz,soma_proportional/bvh/230906/walk_ff_start_360_R_fast_001__A447.bvh,soma_shapes/soma_proportion_fit_mhr_params/A447.npz,g1/csv/230906/walk_ff_start_360_R_fast_001__A447.csv,walk_ff_start_360_R_fast_001,A447,walk_ff_start_360_R_fast_001,230906,_1,walk_ff_start_360_R_fast,start walking backward at a fast pace,"at the beginning, person starts walking backward at a fast pace, moving in the opposite direction with high speed","it starts with character walking at a rapid speed, moving in the reverse direction","A person is walking backward at a fast pace, taking short steps.",start fast paced walk back over right shoulder while facing forward,start fast walk back over right shoulder,walk ff start 360 fast,neutral,neutral,walking,standing,1,0,0,0,0,A447,M,171,29,141,30,92,133,174,98,28,56,12,63,27,F,actor
30
+ jog_arc_cw_start_R_001__A093,jog_arc_cw_loop_004__A093,526,Locomotion,Basic Locomotion Neutral,1.0,False,soma_uniform/bvh/230102/jog_arc_cw_loop_004__A093.bvh,soma_shapes/soma_base_fit_mhr_params.npz,soma_proportional/bvh/230102/jog_arc_cw_loop_004__A093.bvh,soma_shapes/soma_proportion_fit_mhr_params/A093.npz,g1/csv/230102/jog_arc_cw_loop_004__A093.csv,jog_arc_cw_start_R_001,A093,jog_arc_cw_loop_004,230102,_1,jog_arc_cw_start_R,start jogging in a clockwise motion in an arc,"begin jogging in a clockwise arc, starting with the right foot","with right foot in front, jog in a clockwise circle movement","A person is standing straight, facing forward, and does not move.","start right foot, move forward in a circle, clockwise, jog ",start jogging clockwise in an arc,jog arc cw start,neutral,neutral,jogging,standing,1,1,0,0,0,A093,T,188,32,157,36,103,151,192,109,30,56,10,89,24,M,actor
31
+ jog_ff_start_360_R_normal_pace_001__A451,jog_ff_start_360_R_normal_pace_001__A451,777,Locomotion,Basic Locomotion Neutral,1.0,False,soma_uniform/bvh/230907/jog_ff_start_360_R_normal_pace_001__A451.bvh,soma_shapes/soma_base_fit_mhr_params.npz,soma_proportional/bvh/230907/jog_ff_start_360_R_normal_pace_001__A451.bvh,soma_shapes/soma_proportion_fit_mhr_params/A451.npz,g1/csv/230907/jog_ff_start_360_R_normal_pace_001__A451.csv,jog_ff_start_360_R_normal_pace_001,A451,jog_ff_start_360_R_normal_pace_001,230907,_1,jog_ff_start_360_R_normal_pace,start jogging backward at a normal pace,"at the beginning, person starts jogging backward at a normal pace, running in reverse with a moderate speed","it starts with character jogging at a regular pace, moving backward with a moderate speed",A person standing upright jogs backwards with small steps at a steady pace.,start normal paced jog back over right shoulder while facing forward,start normal pace jog back over right shoulder,jog ff start 360 normal pace,neutral,neutral,jogging,standing,1,0,0,0,0,A451,M,167,27,136,29,90,131,165,94,26,51,10,50,24,F,actor
32
+ jog_arc_cw_start_R_normal_pace_001__A451,jog_arc_cw_start_R_normal_pace_001__A451,565,Locomotion,Basic Locomotion Neutral,1.0,False,soma_uniform/bvh/230907/jog_arc_cw_start_R_normal_pace_001__A451.bvh,soma_shapes/soma_base_fit_mhr_params.npz,soma_proportional/bvh/230907/jog_arc_cw_start_R_normal_pace_001__A451.bvh,soma_shapes/soma_proportion_fit_mhr_params/A451.npz,g1/csv/230907/jog_arc_cw_start_R_normal_pace_001__A451.csv,jog_arc_cw_start_R_normal_pace_001,A451,jog_arc_cw_start_R_normal_pace_001,230907,_1,jog_arc_cw_start_R_normal_pace,start jogging in an arc at a normal pace,"at the beginning, person starts jogging in an arc at a normal pace, running in a curved path","it begins with character jogging at a regular pace, moving in a curved path",A person jogs in a clockwise circle at a steady pace.,start normal paced jog in clockwise circle starting with right leg,start normal pace jog in clockwise circle,jog arc cw start normal pace,neutral,neutral,jogging,standing,1,0,0,0,0,A451,M,167,27,136,29,90,131,165,94,26,51,10,50,24,F,actor
33
+ walk_ff_start_360_R_normal_pace_001__A445,walk_ff_start_360_R_normal_pace_001__A445,846,Locomotion,Basic Locomotion Neutral,1.0,False,soma_uniform/bvh/230905/walk_ff_start_360_R_normal_pace_001__A445.bvh,soma_shapes/soma_base_fit_mhr_params.npz,soma_proportional/bvh/230905/walk_ff_start_360_R_normal_pace_001__A445.bvh,soma_shapes/soma_proportion_fit_mhr_params/A445.npz,g1/csv/230905/walk_ff_start_360_R_normal_pace_001__A445.csv,walk_ff_start_360_R_normal_pace_001,A445,walk_ff_start_360_R_normal_pace_001,230905,_2,walk_ff_start_360_R_normal_pace,start walking backward at a normal pace,"at the beginning, person starts walking backward at a normal pace, moving in the opposite direction","it starts with character walking at a regular speed, moving in the reverse direction",A person is walking backward at a consistent pace.,start normal paced walk back over right shoulder while facing forward,start normal pace walk back over right shoulder,walk ff start 360 normal pace,neutral,neutral,walking,standing,1,0,0,0,0,A445,M,184,30,150,33,100,149,187,99,28,57,10,74,27,M,actor
34
+ walk_arc_cw_start_R_normal_pace_001__A443,walk_arc_cw_start_R_normal_pace_001__A443,1122,Locomotion,Basic Locomotion Neutral,1.0,False,soma_uniform/bvh/230905/walk_arc_cw_start_R_normal_pace_001__A443.bvh,soma_shapes/soma_base_fit_mhr_params.npz,soma_proportional/bvh/230905/walk_arc_cw_start_R_normal_pace_001__A443.bvh,soma_shapes/soma_proportion_fit_mhr_params/A443.npz,g1/csv/230905/walk_arc_cw_start_R_normal_pace_001__A443.csv,walk_arc_cw_start_R_normal_pace_001,A443,walk_arc_cw_start_R_normal_pace_001,230905,_2,walk_arc_cw_start_R_normal_pace,start walking in an arc at a normal pace,"at the beginning, person starts walking in an arc at a normal pace, moving in a curved path","it begins with character walking at a regular speed, moving in a curved path",A person is walking in a clockwise circular direction at a normal pace.,start normal paced walk in clockwise circle starting with right leg,start normal pace walk in clockwise circle,walk arc cw start normal pace,neutral,neutral,walking,standing,1,0,0,0,0,A443,M,167,27,136,29,90,131,165,94,26,51,10,50,24,F,actor
35
+ walk_ff_start_360_R_very_fast_001__A447,walk_ff_start_360_R_very_fast_001__A447,475,Locomotion,Basic Locomotion Neutral,1.0,False,soma_uniform/bvh/230906/walk_ff_start_360_R_very_fast_001__A447.bvh,soma_shapes/soma_base_fit_mhr_params.npz,soma_proportional/bvh/230906/walk_ff_start_360_R_very_fast_001__A447.bvh,soma_shapes/soma_proportion_fit_mhr_params/A447.npz,g1/csv/230906/walk_ff_start_360_R_very_fast_001__A447.csv,walk_ff_start_360_R_very_fast_001,A447,walk_ff_start_360_R_very_fast_001,230906,_1,walk_ff_start_360_R_very_fast,start walking backward at a very fast pace,individual starts walking backward at a very fast pace,person begins a rapid walk in the backward direction at an extremely high speed,A person standing on the ground is quickly walking in the backward direction while facing forward.,start very fast paced walk back over right shoulder while facing forward,start very fast walk back over right shoulder,walk ff start 360 very fast,neutral,neutral,walking,standing,1,0,0,0,0,A447,M,171,29,141,30,92,133,174,98,28,56,12,63,27,F,actor
36
+ walk_arc_cw_start_R_very_fast_002__A447,walk_arc_cw_start_R_very_fast_002__A447,712,Locomotion,Basic Locomotion Neutral,1.0,False,soma_uniform/bvh/230906/walk_arc_cw_start_R_very_fast_002__A447.bvh,soma_shapes/soma_base_fit_mhr_params.npz,soma_proportional/bvh/230906/walk_arc_cw_start_R_very_fast_002__A447.bvh,soma_shapes/soma_proportion_fit_mhr_params/A447.npz,g1/csv/230906/walk_arc_cw_start_R_very_fast_002__A447.csv,walk_arc_cw_start_R_very_fast_002,A447,walk_arc_cw_start_R_very_fast_002,230906,_1,walk_arc_cw_start_R_very_fast,start walking in an arc at a very fast pace,individual starts walking in an arc at a very fast pace,person begins a fast-paced walk in a curved pattern at an extremely high speed,A person walks in a clockwise circle at a fast pace.,start very fast walk in clockwise circle starting with right leg,start very fast walk in clockwise circle,walk arc cw start very fast,neutral,neutral,walking,standing,1,0,0,0,0,A447,M,171,29,141,30,92,133,174,98,28,56,12,63,27,F,actor
37
+ jog_arc_cw_stop_R_fast_001__A452,jog_arc_cw_stop_R_fast_001__A452,569,Locomotion,Basic Locomotion Neutral,1.0,False,soma_uniform/bvh/230907/jog_arc_cw_stop_R_fast_001__A452.bvh,soma_shapes/soma_base_fit_mhr_params.npz,soma_proportional/bvh/230907/jog_arc_cw_stop_R_fast_001__A452.bvh,soma_shapes/soma_proportion_fit_mhr_params/A452.npz,g1/csv/230907/jog_arc_cw_stop_R_fast_001__A452.csv,jog_arc_cw_stop_R_fast_001,A452,jog_arc_cw_stop_R_fast_001,230907,_1,jog_arc_cw_stop_R_fast,stop jogging in an arc at a fast pace,person ends their fast-paced jog in an arc,individual stops jogging in a curved path at a high speed,"A person jogs quickly in a clockwise circle, then stops.",stop fast jog in clockwise circle ending with right leg,stop fast jog in clockwise circle,jog arc cw stop fast,neutral,neutral,jogging,standing,1,0,0,0,0,A452,M,171,29,141,30,92,133,174,98,28,56,12,63,27,F,actor
38
+ walk_arc_cw_stop_R_fast_002__A447,walk_arc_cw_stop_R_fast_002__A447,739,Locomotion,Basic Locomotion Neutral,1.0,False,soma_uniform/bvh/230906/walk_arc_cw_stop_R_fast_002__A447.bvh,soma_shapes/soma_base_fit_mhr_params.npz,soma_proportional/bvh/230906/walk_arc_cw_stop_R_fast_002__A447.bvh,soma_shapes/soma_proportion_fit_mhr_params/A447.npz,g1/csv/230906/walk_arc_cw_stop_R_fast_002__A447.csv,walk_arc_cw_stop_R_fast_002,A447,walk_arc_cw_stop_R_fast_002,230906,_1,walk_arc_cw_stop_R_fast,stop walking in an arc at a fast pace,person ends their fast-paced walk in a curved pattern,the individual stops walking in an arc at a high speed,A person walks at a fast pace in a clockwise circle and then stops and stands still.,stop fast walk in clockwise circle ending with right leg,stop fast walk in clockwise circle,walk arc cw stop fast,neutral,neutral,walking,standing,1,0,0,0,0,A447,M,171,29,141,30,92,133,174,98,28,56,12,63,27,F,actor
39
+ jog_arc_cw_stop_R_very_fast_001__A455,jog_arc_cw_stop_R_very_fast_001__A455,486,Locomotion,Basic Locomotion Neutral,1.0,False,soma_uniform/bvh/230907/jog_arc_cw_stop_R_very_fast_001__A455.bvh,soma_shapes/soma_base_fit_mhr_params.npz,soma_proportional/bvh/230907/jog_arc_cw_stop_R_very_fast_001__A455.bvh,soma_shapes/soma_proportion_fit_mhr_params/A455.npz,g1/csv/230907/jog_arc_cw_stop_R_very_fast_001__A455.csv,jog_arc_cw_stop_R_very_fast_001,A455,jog_arc_cw_stop_R_very_fast_001,230907,_2,jog_arc_cw_stop_R_very_fast,stop running in an arc at a very fast pace,person is walking forward slowly with a happy expression,character walks happily in a slow pace towards the front,A person quickly runs in a circular path in a clockwise direction towards their right and halts abruptly.,stop very fast jog in clockwise circle ending with right leg,stop very fast jog in clockwise circle,jog arc cw stop very fast,neutral,neutral,jogging,standing,1,0,0,0,0,A455,M,172,26,140,33,91,131,172,95,27,51,10,68,28,F,actor
40
+ walk_ff_stop_360_R_very_fast_002__A447,walk_ff_stop_360_R_very_fast_002__A447,668,Locomotion,Basic Locomotion Neutral,1.0,False,soma_uniform/bvh/230906/walk_ff_stop_360_R_very_fast_002__A447.bvh,soma_shapes/soma_base_fit_mhr_params.npz,soma_proportional/bvh/230906/walk_ff_stop_360_R_very_fast_002__A447.bvh,soma_shapes/soma_proportion_fit_mhr_params/A447.npz,g1/csv/230906/walk_ff_stop_360_R_very_fast_002__A447.csv,walk_ff_stop_360_R_very_fast_002,A447,walk_ff_stop_360_R_very_fast_002,230906,_1,walk_ff_stop_360_R_very_fast,stop walking backward at a very fast pace,person stops their fast-paced walk backward at a very high speed,character ends their quick walk backward at an extremely rapid pace,"A person walks back fast, facing forward, stops, and stands idle.",stop very fast paced walk back over right shoulder while facing forward,stop very fast walk back over right shoulder,walk ff stop 360 very fast,neutral,neutral,walking,standing,1,0,0,0,0,A447,M,171,29,141,30,92,133,174,98,28,56,12,63,27,F,actor
41
+ touch_toes_standing_R_001__A533,touch_toes_standing_R_001__A533,437,Locomotion,Basic Locomotion Neutral,1.0,False,soma_uniform/bvh/240327/touch_toes_standing_R_001__A533.bvh,soma_shapes/soma_base_fit_mhr_params.npz,soma_proportional/bvh/240327/touch_toes_standing_R_001__A533.bvh,soma_shapes/soma_proportion_fit_mhr_params/A533.npz,g1/csv/240327/touch_toes_standing_R_001__A533.csv,touch_toes_standing_R_001,A533,touch_toes_standing_R_001,240327,_1,touch_toes_standing_R,touch your toes while standing,touch your toes,bend over and touch your toes,A person gradually bends forward and touches their toes with their hands while standing.,person standing bends over forward and touches their toes,touch your toes while standing,touch toes standing,neutral,neutral,action,standing,1,0,0,0,0,A533,M,171,29,141,35,90,134,172,95,26,54,11,71,27,M,actor
42
+ jump_ff_360_R_002__A045,jump_ff_360_002__A045,780,Locomotion,Baseline,1.0,False,soma_uniform/bvh/221010/jump_ff_360_002__A045.bvh,soma_shapes/soma_base_fit_mhr_params.npz,soma_proportional/bvh/221010/jump_ff_360_002__A045.bvh,soma_shapes/soma_proportion_fit_mhr_params/A045.npz,g1/csv/221010/jump_ff_360_002__A045.csv,jump_ff_360_R_002,A045,jump_ff_360_002,221010,_1,jump_ff_360_R,character performs a small jump while moving backward,"begin by moving the right foot, then jump backward",person takes a small jump backward,A person jumps backward while facing forward.,"start right foot, move backward, small jump",jump backwards,jump ff 360,neutral,neutral,jumping,standing,1,1,0,0,0,A045,S,156,25,130,39,82,123,154,87,28,45,8,59,22,F,general
43
+ crouch_idle_002__A148,idle_crouch_loop_002__A148,891,Locomotion,Advanced Locomotion,1.0,False,soma_uniform/bvh/230125/idle_crouch_loop_002__A148.bvh,soma_shapes/soma_base_fit_mhr_params.npz,soma_proportional/bvh/230125/idle_crouch_loop_002__A148.bvh,soma_shapes/soma_proportion_fit_mhr_params/A148.npz,g1/csv/230125/idle_crouch_loop_002__A148.csv,crouch_idle_002,A148,idle_crouch_loop_002,230125,_2,crouch_idle,assume a crouching position with feet almost parallel,assumes a squatting posture with nearly parallel feet in an open position,person is crouching down with their feet in a parallel and open position,A person is sitting in a crouched posture.,"feet almost parallel in open position, talent crouching",crouch position,crouch idle,neutral,neutral,walking,crouching,0,0,0,0,0,A148,M,181,29,152,35,99,145,188,103,29,58,10,73,25,M,actor
44
+ high_jump_R_opt_1_002__A280,high_jump_R_002__A280,376,Locomotion,Baseline,1.0,False,soma_uniform/bvh/230316/high_jump_R_002__A280.bvh,soma_shapes/soma_base_fit_mhr_params.npz,soma_proportional/bvh/230316/high_jump_R_002__A280.bvh,soma_shapes/soma_proportion_fit_mhr_params/A280.npz,g1/csv/230316/high_jump_R_002__A280.csv,high_jump_R_opt_1_002,A280,high_jump_R_002,230316,_1,high_jump_R_opt_1,high jump over a moving pole in one place,making high jump over a moving pole stationary,performer makes high jump in one place over a moving pole,A person energetically does a high jump while facing forward.,"jump high over moving pole, jump high in one place",jumping high,high jump opt 1,neutral,neutral,jumping,standing,1,1,0,0,0,A280,M,171,27,141,32,93,134,170,99,27,50,9,68,25,M,actor
45
+ crouch_idle_right_R_002__A194,idle_crouch_right_loop_R_002__A194,1905,Locomotion,Advanced Locomotion,1.0,False,soma_uniform/bvh/230215/idle_crouch_right_loop_R_002__A194.bvh,soma_shapes/soma_base_fit_mhr_params.npz,soma_proportional/bvh/230215/idle_crouch_right_loop_R_002__A194.bvh,soma_shapes/soma_proportion_fit_mhr_params/A194.npz,g1/csv/230215/idle_crouch_right_loop_R_002__A194.csv,crouch_idle_right_R_002,A194,idle_crouch_right_loop_R_002,230215,_1,crouch_idle_right_R,character with right foot forward assumes a crouching position leaning on the left foot,stand in a crouching position with right foot forward,maintain a crouching position with the right leg slightly extended forward,"A person maintains a crouched position with their right leg forward, making slight movements with their forearms.","right foot slightly forward, body weight on left foot, talent crouching",crouch position right leg forward,crouch idle right,neutral,neutral,crouching idle,crouching,0,0,0,0,0,A194,M,178,29,146,29,91,138,173,98,32,53,10,90,25,M,actor
46
+ walk_ff_start_315_R_002__A049,walk_ff_start_315_002__A049,760,Locomotion,Basic Locomotion Neutral,1.0,False,soma_uniform/bvh/221011/walk_ff_start_315_002__A049.bvh,soma_shapes/soma_base_fit_mhr_params.npz,soma_proportional/bvh/221011/walk_ff_start_315_002__A049.bvh,soma_shapes/soma_proportion_fit_mhr_params/A049.npz,g1/csv/221011/walk_ff_start_315_002__A049.csv,walk_ff_start_315_R_002,A049,walk_ff_start_315_002,221011,_1,walk_ff_start_315_R,walking backward starting with right foot and moving diagonally with left foot crossing behind,"begin by moving right foot, then move backward diagonally crossing with left foot",person takes a right diagonal step backward,A person starts walking swiftly toward the back right diagonal in a crosswise manner at a moderate pace.,"start right foot, move backward diagonally, left foot back crossing, walk backward diagonally",start walking back right diagonal,walk ff start 315,neutral,neutral,walking,standing,1,0,0,0,0,A049,S,156,23,130,40,84,120,154,88,24,47,9,50,25,F,general
47
+ jog_arc_ccw_stop_002__A029,arc_jog_left_stop_002__A029,716,Locomotion,Basic Locomotion Neutral,1.0,False,soma_uniform/bvh/220720/arc_jog_left_stop_002__A029.bvh,soma_shapes/soma_base_fit_mhr_params.npz,soma_proportional/bvh/220720/arc_jog_left_stop_002__A029.bvh,soma_shapes/soma_proportion_fit_mhr_params/A029.npz,g1/csv/220720/arc_jog_left_stop_002__A029.csv,jog_arc_ccw_stop_002,A029,arc_jog_left_stop_002,220720,_1,jog_arc_ccw_stop,"person jogs in a counterclockwise circle, then abruptly stops",person stops jogging in a counterclockwise arc,stop jogging in a counterclockwise circle,A person jogs actively towards left in a semicircle.,"jogging in a circle, counterclockwise, stop",stop jogging counterclockwise in an arc,jog arc ccw stop,neutral,neutral,jogging,standing,1,1,0,0,0,A029,S,155,24,127,31,85,120,158,80,22,43,8,41,22,F,general
48
+ walk_ff_stop_360_R_002__A049,walk_ff_stop_360_002__A049,829,Locomotion,Basic Locomotion Neutral,1.0,False,soma_uniform/bvh/221011/walk_ff_stop_360_002__A049.bvh,soma_shapes/soma_base_fit_mhr_params.npz,soma_proportional/bvh/221011/walk_ff_stop_360_002__A049.bvh,soma_shapes/soma_proportion_fit_mhr_params/A049.npz,g1/csv/221011/walk_ff_stop_360_002__A049.csv,walk_ff_stop_360_R_002,A049,walk_ff_stop_360_002,221011,_1,walk_ff_stop_360_R,person stops and walks backwards,stop the action of walking backwards,"individual ends their backward walk, starting with their left foot",The person stops walking backwards.,"end left foot, move backward, walk backward",stop walking backwards,walk ff stop 360,neutral,neutral,walking,standing,1,0,0,0,0,A049,S,156,23,130,40,84,120,154,88,24,47,9,50,25,F,general
49
+ crouch_ff_stop_180_R_002__A147,crouch_ff_stop_180_R_002__A147,734,Locomotion,Advanced Locomotion,1.0,False,soma_uniform/bvh/230125/crouch_ff_stop_180_R_002__A147.bvh,soma_shapes/soma_base_fit_mhr_params.npz,soma_proportional/bvh/230125/crouch_ff_stop_180_R_002__A147.bvh,soma_shapes/soma_proportion_fit_mhr_params/A147.npz,g1/csv/230125/crouch_ff_stop_180_R_002__A147.csv,crouch_ff_stop_180_R_002,A147,crouch_ff_stop_180_R_002,230125,_2,crouch_ff_stop_180_R,walking in a forwards direction while in a crouching position,a crouching character stops and faces forward after moving forward,start walking forward while maintaining a crouch position,A person walking forward in a crouching position stops walking and stands in the same stance.,"end left foot, move forward, talent crouching, walking in crouch",stopping crouch facing forward,crouch ff stop 180,neutral,neutral,walking,crouching,1,1,0,0,0,A147,M,183,28,150,30,92,143,182,100,31,57,10,84,26,M,actor
50
+ turn_jump_000_002__A022,turn_jump_0000_002__A022,326,Locomotion,Basic Locomotion Neutral,1.0,False,soma_uniform/bvh/220713/turn_jump_0000_002__A022.bvh,soma_shapes/soma_base_fit_mhr_params.npz,soma_proportional/bvh/220713/turn_jump_0000_002__A022.bvh,soma_shapes/soma_proportion_fit_mhr_params/A022.npz,g1/csv/220713/turn_jump_0000_002__A022.csv,turn_jump_000_002,A022,turn_jump_0000_002,220713,_1,turn_jump_000,jumping back over the left shoulder with a full spin,a jump over the left shoulder with a 180-degree turn,person leaps one step while rotating 180 degrees over the left shoulder,A person quickly jumps back over their left shoulder and then stands.,rotate 180 degrees over your left shoulder while jumping one step,turn jump back over left shoulder,turn jump 000,neutral,neutral,"jumping, turning",standing,1,1,0,0,0,A022,T,180,25,152,32,93,135,174,105,28,51,7,62,30,F,general
51
+ jog_ff_start_180_R_002__A021,jog_forward_start_002__A021,608,Locomotion,Baseline,1.0,False,soma_uniform/bvh/220713/jog_forward_start_002__A021.bvh,soma_shapes/soma_base_fit_mhr_params.npz,soma_proportional/bvh/220713/jog_forward_start_002__A021.bvh,soma_shapes/soma_proportion_fit_mhr_params/A021.npz,g1/csv/220713/jog_forward_start_002__A021.csv,jog_ff_start_180_R_002,A021,jog_forward_start_002,220713,_1,jog_ff_start_180_R,start jogging forward,start a forward jog,initiate a forward jog,A person standing on the ground steadily jogs in the forward direction and then stands on the ground in an upright position.,"start right foot, move forward, jog",start jog forward,jog ff start 180,neutral,neutral,jogging,standing,1,1,0,0,0,A021,S,155,23,129,30,85,121,152,85,24,44,7,41,29,F,general
52
+ jog_arc_ccw_start_002__A029,arc_jog_left_start_002__A029,403,Locomotion,Basic Locomotion Neutral,1.0,False,soma_uniform/bvh/220720/arc_jog_left_start_002__A029.bvh,soma_shapes/soma_base_fit_mhr_params.npz,soma_proportional/bvh/220720/arc_jog_left_start_002__A029.bvh,soma_shapes/soma_proportion_fit_mhr_params/A029.npz,g1/csv/220720/arc_jog_left_start_002__A029.csv,jog_arc_ccw_start_002,A029,arc_jog_left_start_002,220720,_1,jog_arc_ccw_start,jogging is initiated in a counterclockwise arc,commence a jog in counterclockwise direction,person starts jogging in a counterclockwise arc,A person standing starts jogging in an anticlockwise arc.,"jogging in a circle, counterclockwise, start",start jogging counterclockwise in an arc,jog arc ccw start,neutral,neutral,jogging,standing,1,1,0,0,0,A029,S,155,24,127,31,85,120,158,80,22,43,8,41,22,F,general
53
+ jog_arc_cw_stop_R_002__A045,jog_arc_cw_stop_002__A045,517,Locomotion,Basic Locomotion Neutral,1.0,False,soma_uniform/bvh/221010/jog_arc_cw_stop_002__A045.bvh,soma_shapes/soma_base_fit_mhr_params.npz,soma_proportional/bvh/221010/jog_arc_cw_stop_002__A045.bvh,soma_shapes/soma_proportion_fit_mhr_params/A045.npz,g1/csv/221010/jog_arc_cw_stop_002__A045.csv,jog_arc_cw_stop_R_002,A045,jog_arc_cw_stop_002,221010,_2,jog_arc_cw_stop_R,"someone stops jogging in a clockwise, arc-like motion",stop and jog clockwise forming a circular path,person finishes jogging in a clockwise arc,"A person is jogging in a clockwise direction, forming an arc after which they stop at a pace.","end left foot, move forward in a circle, clockwise, jog ",stop jogging clockwise in an arc,jog arc cw stop,neutral,neutral,jogging,standing,1,1,0,0,0,A045,S,156,25,130,39,82,123,154,87,28,45,8,59,22,F,general
54
+ walk_ff_stop_180_R_002__A023,walk_forward_stop_002__A023,643,Locomotion,Basic Locomotion Neutral,1.0,False,soma_uniform/bvh/220713/walk_forward_stop_002__A023.bvh,soma_shapes/soma_base_fit_mhr_params.npz,soma_proportional/bvh/220713/walk_forward_stop_002__A023.bvh,soma_shapes/soma_proportion_fit_mhr_params/A023.npz,g1/csv/220713/walk_forward_stop_002__A023.csv,walk_ff_stop_180_R_002,A023,walk_forward_stop_002,220713,_1,walk_ff_stop_180_R,stop moving forward by landing on the left foot while walking,"advance forward and stop with a flourish, landing on your left foot","finish walking forward, ending with the left foot",A person standing is walking in the forward direction at a steady pace.,"end left foot, move forward, walk forward, maximum expression","stop walking forward, finish with left leg",walk ff stop 180,neutral,neutral,walking,standing,1,0,0,0,0,A023,T,190,31,158,42,106,160,195,105,27,54,10,90,46,M,general
55
+ turn_walk_000_L_002__A021,turn_start_walk_0000_002__A021,481,Locomotion,Basic Locomotion Neutral,1.0,False,soma_uniform/bvh/220713/turn_start_walk_0000_002__A021.bvh,soma_shapes/soma_base_fit_mhr_params.npz,soma_proportional/bvh/220713/turn_start_walk_0000_002__A021.bvh,soma_shapes/soma_proportion_fit_mhr_params/A021.npz,g1/csv/220713/turn_start_walk_0000_002__A021.csv,turn_walk_000_L_002,A021,turn_start_walk_0000_002,220713,_1,turn_walk_000_L,figure turns 180 degrees over their left shoulder and then walks forward,a quick turn over the left shoulder followed by a forward walk,character turns 180 degrees over left shoulder and moves forward,A person turns back over their left shoulder and starts walking forward.,"start left foot, turn left 180 degrees over left shoulder, walk forward, move forward","turns back over left shoulder, walks forward",turn walk 000,neutral,neutral,"walking, turning",standing,1,0,0,0,0,A021,S,155,23,129,30,85,121,152,85,24,44,7,41,29,F,general
56
+ jog_ff_start_000_002__A025,jog_backward_start_002__A025,680,Locomotion,Basic Locomotion Neutral,1.0,False,soma_uniform/bvh/220714/jog_backward_start_002__A025.bvh,soma_shapes/soma_base_fit_mhr_params.npz,soma_proportional/bvh/220714/jog_backward_start_002__A025.bvh,soma_shapes/soma_proportion_fit_mhr_params/A025.npz,g1/csv/220714/jog_backward_start_002__A025.csv,jog_ff_start_000_002,A025,jog_backward_start_002,220714,_1,jog_ff_start_000,"start a backwards jog, leading with the left leg","jog backward, starting with the left leg","person performing a backwards jog, facing forward, initiating movement with the left leg",A standing person starts jogging backward steadily by taking medium-length strides while facing forward.,"face forward, jogging back, left leg first, start",begin jog backwards,jog ff start 000,neutral,neutral,jogging,standing,1,1,0,0,0,A025,T,199,31,165,44,103,157,201,113,28,58,10,102,19,M,general
57
+ jump_ff_000_002__A021,jump_backward_002__A021,624,Locomotion,Basic Locomotion Neutral,1.0,False,soma_uniform/bvh/220713/jump_backward_002__A021.bvh,soma_shapes/soma_base_fit_mhr_params.npz,soma_proportional/bvh/220713/jump_backward_002__A021.bvh,soma_shapes/soma_proportion_fit_mhr_params/A021.npz,g1/csv/220713/jump_backward_002__A021.csv,jump_ff_000_002,A021,jump_backward_002,220713,_1,jump_ff_000,"jumping back, landing on the right foot",jumping backwards and landing on the right leg,"a jump backwards led by the left foot, while constantly looking forwards",A person jumps backward and then stands in an upright posture.,"jumping back, left foot first, head looking forward","jump backwards, finish with right leg",jump ff 000,neutral,neutral,jumping,standing,1,1,0,0,0,A021,S,155,23,129,30,85,121,152,85,24,44,7,41,29,F,general
58
+ jog_ff_start_315_R_002__A050,jog_ff_start_315_002__A050,438,Locomotion,Basic Locomotion Neutral,1.0,False,soma_uniform/bvh/221011/jog_ff_start_315_002__A050.bvh,soma_shapes/soma_base_fit_mhr_params.npz,soma_proportional/bvh/221011/jog_ff_start_315_002__A050.bvh,soma_shapes/soma_proportion_fit_mhr_params/A050.npz,g1/csv/221011/jog_ff_start_315_002__A050.csv,jog_ff_start_315_R_002,A050,jog_ff_start_315_002,221011,_1,jog_ff_start_315_R,person starts jogging backward diagonally with the right foot,person jogs backward diagonally starting with the right foot,person starts a diagonal jog backward,A person swiftly jogs backward along the back-right diagonal.,"start right foot, move backward diagonally, left foot back crossing, jog",start jog back right diagonally,jog ff start 315,neutral,neutral,jogging,standing,1,1,0,0,0,A050,T,182,26,152,40,99,140,180,105,24,53,10,75,31,F,general
59
+ turn_jump_090_002__A025,turn_jump_0090_002__A025,387,Locomotion,Basic Locomotion Neutral,1.0,False,soma_uniform/bvh/220714/turn_jump_0090_002__A025.bvh,soma_shapes/soma_base_fit_mhr_params.npz,soma_proportional/bvh/220714/turn_jump_0090_002__A025.bvh,soma_shapes/soma_proportion_fit_mhr_params/A025.npz,g1/csv/220714/turn_jump_0090_002__A025.csv,turn_jump_090_002,A025,turn_jump_0090_002,220714,_1,turn_jump_090,individual performing a leftward turning jump step,a one-step jump with a 90-degree rotation over the left shoulder,individual turns 90 degrees over their left shoulder while jumping one step,A person swiftly pivots and jumps left with agility.,rotate 90 degrees over your left shoulder while jumping one step,turn jump left,turn jump 090,neutral,neutral,"jumping, turning",standing,1,1,0,0,0,A025,T,199,31,165,44,103,157,201,113,28,58,10,102,19,M,general
60
+ crouch_turn_360_R_002__A125,turn_crouch_360_002__A125,433,Locomotion,Advanced Locomotion,1.0,False,soma_uniform/bvh/230119/turn_crouch_360_002__A125.bvh,soma_shapes/soma_base_fit_mhr_params.npz,soma_proportional/bvh/230119/turn_crouch_360_002__A125.bvh,soma_shapes/soma_proportion_fit_mhr_params/A125.npz,g1/csv/230119/turn_crouch_360_002__A125.csv,crouch_turn_360_R_002,A125,turn_crouch_360_002,230119,_3,crouch_turn_360_R,character crouches and turns back over the right shoulder,character makes a 180 degree turn over the right shoulder while crouching,complete a turn back over the right shoulder while keeping a crouch,A person in a crouched position quickly turns back over the right shoulder and then sits facing backward and resting arms on thighs.,"start right foot, turn right 180 degrees over right shoulder, 3 steps, talent crouching, walking in crouch",turning back over right shoulder while crouching,crouch turn 360,neutral,neutral,turning,crouching,0,0,0,0,0,A125,M,175,26,147,30,96,137,180,102,29,53,8,64,28,F,actor
61
+ turn_walk_045_L_002__A023,turn_start_walk_0045_002__A023,423,Locomotion,Basic Locomotion Neutral,1.0,False,soma_uniform/bvh/220713/turn_start_walk_0045_002__A023.bvh,soma_shapes/soma_base_fit_mhr_params.npz,soma_proportional/bvh/220713/turn_start_walk_0045_002__A023.bvh,soma_shapes/soma_proportion_fit_mhr_params/A023.npz,g1/csv/220713/turn_start_walk_0045_002__A023.csv,turn_walk_045_L_002,A023,turn_start_walk_0045_002,220713,_1,turn_walk_045_L,character takes strides forward after a 135-degree left turn over the left shoulder,"character makes a 135 degree left turn, walking forward","person takes a left step, turns left 135 degrees and walks forward",A person turns around towards their left back diagonal and starts walking steadily.,"start left foot, turn left 135 degrees over left shoulder, walk forward diagonally",turns left walks forward,turn walk 045,neutral,neutral,"walking, turning",standing,1,0,0,0,0,A023,T,190,31,158,42,106,160,195,105,27,54,10,90,46,M,general
62
+ jump_twice_R_002__A533,jump_twice_R_002__A533,394,Locomotion,Basic Locomotion Neutral,1.0,False,soma_uniform/bvh/240327/jump_twice_R_002__A533.bvh,soma_shapes/soma_base_fit_mhr_params.npz,soma_proportional/bvh/240327/jump_twice_R_002__A533.bvh,soma_shapes/soma_proportion_fit_mhr_params/A533.npz,g1/csv/240327/jump_twice_R_002__A533.csv,jump_twice_R_002,A533,jump_twice_R_002,240327,_1,jump_twice_R,jump and jump again,person jumps forward twice,two big jumps one after the other,A person energetically jumps twice in a row.,person jumps forward right leg first and then jumps again,jump twice in a row,jump twice,neutral,neutral,jumping,standing,1,1,0,0,1,A533,M,171,29,141,35,90,134,172,95,26,54,11,71,27,M,actor
63
+ relaxed_fast_walk_ff_180_002__A005,walk_fast_forward_relax_002__A005,2178,Locomotion,Basic Locomotion Styles,0.0,False,soma_uniform/bvh/210707/walk_fast_forward_relax_002__A005.bvh,soma_shapes/soma_base_fit_mhr_params.npz,soma_proportional/bvh/210707/walk_fast_forward_relax_002__A005.bvh,soma_shapes/soma_proportion_fit_mhr_params/A005.npz,g1/csv/210707/walk_fast_forward_relax_002__A005.csv,relaxed_fast_walk_ff_180_002,A005,walk_fast_forward_relax_002,210707,_1,relaxed_fast_walk_ff_180,a casual individual briskly walking forward with a sense of calmness,"person walks at a brisk pace forward, maintaining an open and relaxed posture","walk forward quickly, doing so with a relaxed, open posture","A person with arms raised sideways lowers them and walks forward quickly, turning around during the walk and then stands with their arms raised sideways.","person walks forward, high energy, open position, calm",relaxed fast walk facing forward,relaxed fast walk ff 180,relaxed,neutral,walking,standing,1,0,0,0,0,A005,S,152,25,129,32,85,120,153,92,28,45,7,50,23,F,general
64
+ crouch_ff_stop_360_R_002__A126,crouch_ff_stop_360_002__A126,819,Locomotion,Advanced Locomotion,1.0,False,soma_uniform/bvh/230119/crouch_ff_stop_360_002__A126.bvh,soma_shapes/soma_base_fit_mhr_params.npz,soma_proportional/bvh/230119/crouch_ff_stop_360_002__A126.bvh,soma_shapes/soma_proportion_fit_mhr_params/A126.npz,g1/csv/230119/crouch_ff_stop_360_002__A126.csv,crouch_ff_stop_360_R_002,A126,crouch_ff_stop_360_002,230119,_3,crouch_ff_stop_360_R,person stops crouched walk and turns over their right shoulder,"a crouching person, moves backward and stops over the right shoulder",person crouching walks backwards and stops over their right shoulder,The crouched person walks backward while facing forward with arms swaying.,"end left foot, move backward, talent crouching, walking in crouch",stopping crouch back over right shoulder,crouch ff stop 360,neutral,neutral,walking,crouching,1,1,0,0,0,A126,M,169,25,136,25,90,130,164,95,31,51,8,63,25,F,actor
65
+ high_jump_full_turn_R_opt_1_002__A477,high_jump_full_turn_R_opt_1_002__A477,427,Locomotion,Advanced Locomotion,1.0,False,soma_uniform/bvh/231011/high_jump_full_turn_R_opt_1_002__A477.bvh,soma_shapes/soma_base_fit_mhr_params.npz,soma_proportional/bvh/231011/high_jump_full_turn_R_opt_1_002__A477.bvh,soma_shapes/soma_proportion_fit_mhr_params/A477.npz,g1/csv/231011/high_jump_full_turn_R_opt_1_002__A477.csv,high_jump_full_turn_R_opt_1_002,A477,high_jump_full_turn_R_opt_1_002,231011,_1,high_jump_full_turn_R_opt_1,turning around during high jump up,the performer turns around while jumping high up,the performer turns around during a high jump,"A person quickly jumps while rotating right, turns around mid-air, and lands on the ground facing forward.",character turns around during a high jump up,turning around during a high jump up and landing,high jump full turn opt 1,neutral,neutral,jumping,standing,0,1,0,0,0,A477,M,168,28,140,28,97,132,170,98,29,52,11,62,28,F,actor
66
+ jump_ff_360_R_003__A045,jump_ff_360_003__A045,655,Locomotion,Baseline,1.0,False,soma_uniform/bvh/221010/jump_ff_360_003__A045.bvh,soma_shapes/soma_base_fit_mhr_params.npz,soma_proportional/bvh/221010/jump_ff_360_003__A045.bvh,soma_shapes/soma_proportion_fit_mhr_params/A045.npz,g1/csv/221010/jump_ff_360_003__A045.csv,jump_ff_360_R_003,A045,jump_ff_360_003,221010,_1,jump_ff_360_R,character performs a small jump while moving backward,"begin by moving the right foot, then jump backward",person takes a small jump backward,A person jumps backward while facing forward.,"start right foot, move backward, small jump",jump backwards,jump ff 360,neutral,neutral,jumping,standing,1,1,0,0,0,A045,S,156,25,130,39,82,123,154,87,28,45,8,59,22,F,general
67
+ crouch_idle_003__A148,idle_crouch_loop_003__A148,831,Locomotion,Advanced Locomotion,1.0,False,soma_uniform/bvh/230125/idle_crouch_loop_003__A148.bvh,soma_shapes/soma_base_fit_mhr_params.npz,soma_proportional/bvh/230125/idle_crouch_loop_003__A148.bvh,soma_shapes/soma_proportion_fit_mhr_params/A148.npz,g1/csv/230125/idle_crouch_loop_003__A148.csv,crouch_idle_003,A148,idle_crouch_loop_003,230125,_2,crouch_idle,assume a crouching position with feet almost parallel,assumes a squatting posture with nearly parallel feet in an open position,person is crouching down with their feet in a parallel and open position,A person is sitting in a crouched posture.,"feet almost parallel in open position, talent crouching",crouch position,crouch idle,neutral,neutral,walking,crouching,0,0,0,0,0,A148,M,181,29,152,35,99,145,188,103,29,58,10,73,25,M,actor
68
+ high_jump_R_opt_1_003__A280,high_jump_R_003__A280,514,Locomotion,Baseline,1.0,False,soma_uniform/bvh/230316/high_jump_R_003__A280.bvh,soma_shapes/soma_base_fit_mhr_params.npz,soma_proportional/bvh/230316/high_jump_R_003__A280.bvh,soma_shapes/soma_proportion_fit_mhr_params/A280.npz,g1/csv/230316/high_jump_R_003__A280.csv,high_jump_R_opt_1_003,A280,high_jump_R_003,230316,_1,high_jump_R_opt_1,high jump over a moving pole in one place,making high jump over a moving pole stationary,performer makes high jump in one place over a moving pole,A person energetically does a high jump while facing forward.,"jump high over moving pole, jump high in one place",jumping high,high jump opt 1,neutral,neutral,jumping,standing,1,1,0,0,0,A280,M,171,27,141,32,93,134,170,99,27,50,9,68,25,M,actor
69
+ crouch_idle_right_R_003__A194,idle_crouch_right_loop_R_003__A194,1225,Locomotion,Advanced Locomotion,1.0,False,soma_uniform/bvh/230215/idle_crouch_right_loop_R_003__A194.bvh,soma_shapes/soma_base_fit_mhr_params.npz,soma_proportional/bvh/230215/idle_crouch_right_loop_R_003__A194.bvh,soma_shapes/soma_proportion_fit_mhr_params/A194.npz,g1/csv/230215/idle_crouch_right_loop_R_003__A194.csv,crouch_idle_right_R_003,A194,idle_crouch_right_loop_R_003,230215,_1,crouch_idle_right_R,character with right foot forward assumes a crouching position leaning on the left foot,stand in a crouching position with right foot forward,maintain a crouching position with the right leg slightly extended forward,"A person maintains a crouched position with their right leg forward, making slight movements with their forearms.","right foot slightly forward, body weight on left foot, talent crouching",crouch position right leg forward,crouch idle right,neutral,neutral,crouching idle,crouching,0,0,0,0,0,A194,M,178,29,146,29,91,138,173,98,32,53,10,90,25,M,actor
70
+ walk_ff_start_315_R_003__A049,walk_ff_start_315_003__A049,800,Locomotion,Basic Locomotion Neutral,1.0,False,soma_uniform/bvh/221011/walk_ff_start_315_003__A049.bvh,soma_shapes/soma_base_fit_mhr_params.npz,soma_proportional/bvh/221011/walk_ff_start_315_003__A049.bvh,soma_shapes/soma_proportion_fit_mhr_params/A049.npz,g1/csv/221011/walk_ff_start_315_003__A049.csv,walk_ff_start_315_R_003,A049,walk_ff_start_315_003,221011,_1,walk_ff_start_315_R,walking backward starting with right foot and moving diagonally with left foot crossing behind,"begin by moving right foot, then move backward diagonally crossing with left foot",person takes a right diagonal step backward,A person starts walking swiftly toward the back right diagonal in a crosswise manner at a moderate pace.,"start right foot, move backward diagonally, left foot back crossing, walk backward diagonally",start walking back right diagonal,walk ff start 315,neutral,neutral,walking,standing,1,0,0,0,0,A049,S,156,23,130,40,84,120,154,88,24,47,9,50,25,F,general
71
+ jog_arc_ccw_stop_003__A029,arc_jog_left_stop_003__A029,533,Locomotion,Basic Locomotion Neutral,1.0,False,soma_uniform/bvh/220720/arc_jog_left_stop_003__A029.bvh,soma_shapes/soma_base_fit_mhr_params.npz,soma_proportional/bvh/220720/arc_jog_left_stop_003__A029.bvh,soma_shapes/soma_proportion_fit_mhr_params/A029.npz,g1/csv/220720/arc_jog_left_stop_003__A029.csv,jog_arc_ccw_stop_003,A029,arc_jog_left_stop_003,220720,_1,jog_arc_ccw_stop,"person jogs in a counterclockwise circle, then abruptly stops",person stops jogging in a counterclockwise arc,stop jogging in a counterclockwise circle,A person jogs actively towards left in a semicircle.,"jogging in a circle, counterclockwise, stop",stop jogging counterclockwise in an arc,jog arc ccw stop,neutral,neutral,jogging,standing,1,1,0,0,0,A029,S,155,24,127,31,85,120,158,80,22,43,8,41,22,F,general
72
+ walk_ff_stop_360_R_003__A049,walk_ff_stop_360_003__A049,198,Locomotion,Basic Locomotion Neutral,1.0,False,soma_uniform/bvh/221011/walk_ff_stop_360_003__A049.bvh,soma_shapes/soma_base_fit_mhr_params.npz,soma_proportional/bvh/221011/walk_ff_stop_360_003__A049.bvh,soma_shapes/soma_proportion_fit_mhr_params/A049.npz,g1/csv/221011/walk_ff_stop_360_003__A049.csv,walk_ff_stop_360_R_003,A049,walk_ff_stop_360_003,221011,_1,walk_ff_stop_360_R,person stops and walks backwards,stop the action of walking backwards,"individual ends their backward walk, starting with their left foot",The person stops walking backwards.,"end left foot, move backward, walk backward",stop walking backwards,walk ff stop 360,neutral,neutral,walking,standing,1,0,0,0,0,A049,S,156,23,130,40,84,120,154,88,24,47,9,50,25,F,general
73
+ crouch_ff_stop_180_R_003__A146,crouch_ff_stop_180_R_003__A146,674,Locomotion,Advanced Locomotion,1.0,False,soma_uniform/bvh/230125/crouch_ff_stop_180_R_003__A146.bvh,soma_shapes/soma_base_fit_mhr_params.npz,soma_proportional/bvh/230125/crouch_ff_stop_180_R_003__A146.bvh,soma_shapes/soma_proportion_fit_mhr_params/A146.npz,g1/csv/230125/crouch_ff_stop_180_R_003__A146.csv,crouch_ff_stop_180_R_003,A146,crouch_ff_stop_180_R_003,230125,_2,crouch_ff_stop_180_R,walking in a forwards direction while in a crouching position,a crouching character stops and faces forward after moving forward,start walking forward while maintaining a crouch position,A crouched person walks forward and then stands for a while.,"end left foot, move forward, talent crouching, walking in crouch",stopping crouch facing forward,crouch ff stop 180,neutral,neutral,walking,crouching,1,1,0,0,0,A146,M,168,27,140,30,95,138,171,90,26,52,8,57,25,F,actor
74
+ turn_jump_000_003__A022,turn_jump_0000_003__A022,465,Locomotion,Basic Locomotion Neutral,1.0,False,soma_uniform/bvh/220713/turn_jump_0000_003__A022.bvh,soma_shapes/soma_base_fit_mhr_params.npz,soma_proportional/bvh/220713/turn_jump_0000_003__A022.bvh,soma_shapes/soma_proportion_fit_mhr_params/A022.npz,g1/csv/220713/turn_jump_0000_003__A022.csv,turn_jump_000_003,A022,turn_jump_0000_003,220713,_1,turn_jump_000,jumping back over the left shoulder with a full spin,a jump over the left shoulder with a 180-degree turn,person leaps one step while rotating 180 degrees over the left shoulder,A person quickly jumps back over their left shoulder and then stands.,rotate 180 degrees over your left shoulder while jumping one step,turn jump back over left shoulder,turn jump 000,neutral,neutral,"jumping, turning",standing,1,1,0,0,0,A022,T,180,25,152,32,93,135,174,105,28,51,7,62,30,F,general
75
+ jog_ff_start_180_R_003__A022,jog_forward_start_003__A022,424,Locomotion,Baseline,1.0,False,soma_uniform/bvh/220713/jog_forward_start_003__A022.bvh,soma_shapes/soma_base_fit_mhr_params.npz,soma_proportional/bvh/220713/jog_forward_start_003__A022.bvh,soma_shapes/soma_proportion_fit_mhr_params/A022.npz,g1/csv/220713/jog_forward_start_003__A022.csv,jog_ff_start_180_R_003,A022,jog_forward_start_003,220713,_1,jog_ff_start_180_R,start jogging forward,start a forward jog,initiate a forward jog,A person standing on the ground steadily jogs in the forward direction and then stands on the ground in an upright position.,"start right foot, move forward, jog",start jog forward,jog ff start 180,neutral,neutral,jogging,standing,1,1,0,0,0,A022,T,180,25,152,32,93,135,174,105,28,51,7,62,30,F,general
76
+ jog_arc_ccw_start_003__A029,arc_jog_left_start_003__A029,512,Locomotion,Basic Locomotion Neutral,1.0,False,soma_uniform/bvh/220720/arc_jog_left_start_003__A029.bvh,soma_shapes/soma_base_fit_mhr_params.npz,soma_proportional/bvh/220720/arc_jog_left_start_003__A029.bvh,soma_shapes/soma_proportion_fit_mhr_params/A029.npz,g1/csv/220720/arc_jog_left_start_003__A029.csv,jog_arc_ccw_start_003,A029,arc_jog_left_start_003,220720,_1,jog_arc_ccw_start,jogging is initiated in a counterclockwise arc,commence a jog in counterclockwise direction,person starts jogging in a counterclockwise arc,A person standing starts jogging in an anticlockwise arc.,"jogging in a circle, counterclockwise, start",start jogging counterclockwise in an arc,jog arc ccw start,neutral,neutral,jogging,standing,1,1,0,0,0,A029,S,155,24,127,31,85,120,158,80,22,43,8,41,22,F,general
77
+ jog_arc_cw_stop_R_003__A047,jog_arc_cw_stop_003__A047,821,Locomotion,Basic Locomotion Neutral,1.0,False,soma_uniform/bvh/221010/jog_arc_cw_stop_003__A047.bvh,soma_shapes/soma_base_fit_mhr_params.npz,soma_proportional/bvh/221010/jog_arc_cw_stop_003__A047.bvh,soma_shapes/soma_proportion_fit_mhr_params/A047.npz,g1/csv/221010/jog_arc_cw_stop_003__A047.csv,jog_arc_cw_stop_R_003,A047,jog_arc_cw_stop_003,221010,_2,jog_arc_cw_stop_R,"someone stops jogging in a clockwise, arc-like motion",stop and jog clockwise forming a circular path,person finishes jogging in a clockwise arc,"A person is jogging in a clockwise arc over the right shoulder, then stops and stands upright.","end left foot, move forward in a circle, clockwise, jog ",stop jogging clockwise in an arc,jog arc cw stop,neutral,neutral,jogging,standing,1,1,0,0,0,A047,T,189,29,156,48,103,153,193,103,29,58,10,100,33,M,general
78
+ walk_ff_stop_180_R_003__A025,walk_forward_stop_003__A025,907,Locomotion,Basic Locomotion Neutral,1.0,False,soma_uniform/bvh/220714/walk_forward_stop_003__A025.bvh,soma_shapes/soma_base_fit_mhr_params.npz,soma_proportional/bvh/220714/walk_forward_stop_003__A025.bvh,soma_shapes/soma_proportion_fit_mhr_params/A025.npz,g1/csv/220714/walk_forward_stop_003__A025.csv,walk_ff_stop_180_R_003,A025,walk_forward_stop_003,220714,_1,walk_ff_stop_180_R,stop moving forward by landing on the left foot while walking,"advance forward and stop with a flourish, landing on your left foot","finish walking forward, ending with the left foot",A person standing is walking in the forward direction at a steady pace.,"end left foot, move forward, walk forward, maximum expression","stop walking forward, finish with left leg",walk ff stop 180,neutral,neutral,walking,standing,1,0,0,0,0,A025,T,199,31,165,44,103,157,201,113,28,58,10,102,19,M,general
79
+ turn_walk_000_L_003__A021,turn_start_walk_0000_003__A021,411,Locomotion,Basic Locomotion Neutral,1.0,False,soma_uniform/bvh/220713/turn_start_walk_0000_003__A021.bvh,soma_shapes/soma_base_fit_mhr_params.npz,soma_proportional/bvh/220713/turn_start_walk_0000_003__A021.bvh,soma_shapes/soma_proportion_fit_mhr_params/A021.npz,g1/csv/220713/turn_start_walk_0000_003__A021.csv,turn_walk_000_L_003,A021,turn_start_walk_0000_003,220713,_1,turn_walk_000_L,figure turns 180 degrees over their left shoulder and then walks forward,a quick turn over the left shoulder followed by a forward walk,character turns 180 degrees over left shoulder and moves forward,A person turns back over their left shoulder and starts walking forward.,"start left foot, turn left 180 degrees over left shoulder, walk forward, move forward","turns back over left shoulder, walks forward",turn walk 000,neutral,neutral,"walking, turning",standing,1,0,0,0,0,A021,S,155,23,129,30,85,121,152,85,24,44,7,41,29,F,general
80
+ jog_ff_start_000_003__A025,jog_backward_start_003__A025,666,Locomotion,Basic Locomotion Neutral,1.0,False,soma_uniform/bvh/220714/jog_backward_start_003__A025.bvh,soma_shapes/soma_base_fit_mhr_params.npz,soma_proportional/bvh/220714/jog_backward_start_003__A025.bvh,soma_shapes/soma_proportion_fit_mhr_params/A025.npz,g1/csv/220714/jog_backward_start_003__A025.csv,jog_ff_start_000_003,A025,jog_backward_start_003,220714,_1,jog_ff_start_000,"start a backwards jog, leading with the left leg","jog backward, starting with the left leg","person performing a backwards jog, facing forward, initiating movement with the left leg",A standing person starts jogging backward steadily by taking medium-length strides while facing forward.,"face forward, jogging back, left leg first, start",begin jog backwards,jog ff start 000,neutral,neutral,jogging,standing,1,1,0,0,0,A025,T,199,31,165,44,103,157,201,113,28,58,10,102,19,M,general
81
+ jump_ff_000_003__A021,jump_backward_003__A021,284,Locomotion,Basic Locomotion Neutral,1.0,False,soma_uniform/bvh/220713/jump_backward_003__A021.bvh,soma_shapes/soma_base_fit_mhr_params.npz,soma_proportional/bvh/220713/jump_backward_003__A021.bvh,soma_shapes/soma_proportion_fit_mhr_params/A021.npz,g1/csv/220713/jump_backward_003__A021.csv,jump_ff_000_003,A021,jump_backward_003,220713,_1,jump_ff_000,"jumping back, landing on the right foot",jumping backwards and landing on the right leg,"a jump backwards led by the left foot, while constantly looking forwards",A person jumps backward and then stands in an upright posture.,"jumping back, left foot first, head looking forward","jump backwards, finish with right leg",jump ff 000,neutral,neutral,jumping,standing,1,1,0,0,0,A021,S,155,23,129,30,85,121,152,85,24,44,7,41,29,F,general
82
+ jog_ff_start_315_R_003__A050,jog_ff_start_315_003__A050,438,Locomotion,Basic Locomotion Neutral,1.0,False,soma_uniform/bvh/221011/jog_ff_start_315_003__A050.bvh,soma_shapes/soma_base_fit_mhr_params.npz,soma_proportional/bvh/221011/jog_ff_start_315_003__A050.bvh,soma_shapes/soma_proportion_fit_mhr_params/A050.npz,g1/csv/221011/jog_ff_start_315_003__A050.csv,jog_ff_start_315_R_003,A050,jog_ff_start_315_003,221011,_1,jog_ff_start_315_R,person starts jogging backward diagonally with the right foot,person jogs backward diagonally starting with the right foot,person starts a diagonal jog backward,A person swiftly jogs backward along the back-right diagonal.,"start right foot, move backward diagonally, left foot back crossing, jog",start jog back right diagonally,jog ff start 315,neutral,neutral,jogging,standing,1,1,0,0,0,A050,T,182,26,152,40,99,140,180,105,24,53,10,75,31,F,general
83
+ turn_jump_090_003__A023,turn_jump_0090_003__A023,735,Locomotion,Basic Locomotion Neutral,1.0,False,soma_uniform/bvh/220713/turn_jump_0090_003__A023.bvh,soma_shapes/soma_base_fit_mhr_params.npz,soma_proportional/bvh/220713/turn_jump_0090_003__A023.bvh,soma_shapes/soma_proportion_fit_mhr_params/A023.npz,g1/csv/220713/turn_jump_0090_003__A023.csv,turn_jump_090_003,A023,turn_jump_0090_003,220713,_1,turn_jump_090,individual performing a leftward turning jump step,a one-step jump with a 90-degree rotation over the left shoulder,individual turns 90 degrees over their left shoulder while jumping one step,A person swiftly pivots and jumps left with agility.,rotate 90 degrees over your left shoulder while jumping one step,turn jump left,turn jump 090,neutral,neutral,"jumping, turning",standing,1,1,0,0,0,A023,T,190,31,158,42,106,160,195,105,27,54,10,90,46,M,general
84
+ crouch_turn_360_R_003__A125,turn_crouch_360_003__A125,533,Locomotion,Advanced Locomotion,1.0,False,soma_uniform/bvh/230119/turn_crouch_360_003__A125.bvh,soma_shapes/soma_base_fit_mhr_params.npz,soma_proportional/bvh/230119/turn_crouch_360_003__A125.bvh,soma_shapes/soma_proportion_fit_mhr_params/A125.npz,g1/csv/230119/turn_crouch_360_003__A125.csv,crouch_turn_360_R_003,A125,turn_crouch_360_003,230119,_3,crouch_turn_360_R,character crouches and turns back over the right shoulder,character makes a 180 degree turn over the right shoulder while crouching,complete a turn back over the right shoulder while keeping a crouch,A person in a crouched position quickly turns back over the right shoulder and then sits facing backward and resting arms on thighs.,"start right foot, turn right 180 degrees over right shoulder, 3 steps, talent crouching, walking in crouch",turning back over right shoulder while crouching,crouch turn 360,neutral,neutral,turning,crouching,0,0,0,0,0,A125,M,175,26,147,30,96,137,180,102,29,53,8,64,28,F,actor
85
+ turn_walk_045_L_003__A023,turn_start_walk_0045_003__A023,516,Locomotion,Basic Locomotion Neutral,1.0,False,soma_uniform/bvh/220713/turn_start_walk_0045_003__A023.bvh,soma_shapes/soma_base_fit_mhr_params.npz,soma_proportional/bvh/220713/turn_start_walk_0045_003__A023.bvh,soma_shapes/soma_proportion_fit_mhr_params/A023.npz,g1/csv/220713/turn_start_walk_0045_003__A023.csv,turn_walk_045_L_003,A023,turn_start_walk_0045_003,220713,_1,turn_walk_045_L,character takes strides forward after a 135-degree left turn over the left shoulder,"character makes a 135 degree left turn, walking forward","person takes a left step, turns left 135 degrees and walks forward",A person turns around towards their left back diagonal and starts walking steadily.,"start left foot, turn left 135 degrees over left shoulder, walk forward diagonally",turns left walks forward,turn walk 045,neutral,neutral,"walking, turning",standing,1,0,0,0,0,A023,T,190,31,158,42,106,160,195,105,27,54,10,90,46,M,general
86
+ jump_twice_R_003__A533,jump_twice_R_003__A533,483,Locomotion,Basic Locomotion Neutral,1.0,False,soma_uniform/bvh/240327/jump_twice_R_003__A533.bvh,soma_shapes/soma_base_fit_mhr_params.npz,soma_proportional/bvh/240327/jump_twice_R_003__A533.bvh,soma_shapes/soma_proportion_fit_mhr_params/A533.npz,g1/csv/240327/jump_twice_R_003__A533.csv,jump_twice_R_003,A533,jump_twice_R_003,240327,_1,jump_twice_R,jump and jump again,person jumps forward twice,two big jumps one after the other,"A person jumps forward twice in a row, and stands in an upright posture, taking a slight pause between the jumps.",person jumps forward right leg first and then jumps again,jump twice in a row,jump twice,neutral,neutral,jumping,standing,1,1,0,0,1,A533,M,171,29,141,35,90,134,172,95,26,54,11,71,27,M,actor
87
+ relaxed_fast_walk_ff_180_003__A005,walk_fast_forward_relax_003__A005,2252,Locomotion,Basic Locomotion Styles,0.0,False,soma_uniform/bvh/210707/walk_fast_forward_relax_003__A005.bvh,soma_shapes/soma_base_fit_mhr_params.npz,soma_proportional/bvh/210707/walk_fast_forward_relax_003__A005.bvh,soma_shapes/soma_proportion_fit_mhr_params/A005.npz,g1/csv/210707/walk_fast_forward_relax_003__A005.csv,relaxed_fast_walk_ff_180_003,A005,walk_fast_forward_relax_003,210707,_1,relaxed_fast_walk_ff_180,a casual individual briskly walking forward with a sense of calmness,"person walks at a brisk pace forward, maintaining an open and relaxed posture","walk forward quickly, doing so with a relaxed, open posture","A person with arms raised sideways lowers them and walks forward quickly, turning around during the walk and then stands with their arms raised sideways.","person walks forward, high energy, open position, calm",relaxed fast walk facing forward,relaxed fast walk ff 180,relaxed,neutral,walking,standing,1,0,0,0,0,A005,S,152,25,129,32,85,120,153,92,28,45,7,50,23,F,general
88
+ crouch_ff_stop_360_R_003__A126,crouch_ff_stop_360_003__A126,893,Locomotion,Advanced Locomotion,1.0,False,soma_uniform/bvh/230119/crouch_ff_stop_360_003__A126.bvh,soma_shapes/soma_base_fit_mhr_params.npz,soma_proportional/bvh/230119/crouch_ff_stop_360_003__A126.bvh,soma_shapes/soma_proportion_fit_mhr_params/A126.npz,g1/csv/230119/crouch_ff_stop_360_003__A126.csv,crouch_ff_stop_360_R_003,A126,crouch_ff_stop_360_003,230119,_3,crouch_ff_stop_360_R,person stops crouched walk and turns over their right shoulder,"a crouching person, moves backward and stops over the right shoulder",person crouching walks backwards and stops over their right shoulder,The crouched person walks backward while facing forward with arms swaying.,"end left foot, move backward, talent crouching, walking in crouch",stopping crouch back over right shoulder,crouch ff stop 360,neutral,neutral,walking,crouching,1,1,0,0,0,A126,M,169,25,136,25,90,130,164,95,31,51,8,63,25,F,actor
89
+ jump_ff_360_R_004__A045,jump_ff_360_004__A045,665,Locomotion,Baseline,1.0,False,soma_uniform/bvh/221010/jump_ff_360_004__A045.bvh,soma_shapes/soma_base_fit_mhr_params.npz,soma_proportional/bvh/221010/jump_ff_360_004__A045.bvh,soma_shapes/soma_proportion_fit_mhr_params/A045.npz,g1/csv/221010/jump_ff_360_004__A045.csv,jump_ff_360_R_004,A045,jump_ff_360_004,221010,_1,jump_ff_360_R,character performs a small jump while moving backward,"begin by moving the right foot, then jump backward",person takes a small jump backward,A person jumps backward while facing forward.,"start right foot, move backward, small jump",jump backwards,jump ff 360,neutral,neutral,jumping,standing,1,1,0,0,0,A045,S,156,25,130,39,82,123,154,87,28,45,8,59,22,F,general
90
+ crouch_idle_004__A244,crouch_idle_004__A244,975,Locomotion,Advanced Locomotion,1.0,False,soma_uniform/bvh/230306/crouch_idle_004__A244.bvh,soma_shapes/soma_base_fit_mhr_params.npz,soma_proportional/bvh/230306/crouch_idle_004__A244.bvh,soma_shapes/soma_proportion_fit_mhr_params/A244.npz,g1/csv/230306/crouch_idle_004__A244.csv,crouch_idle_004,A244,crouch_idle_004,230306,_1,crouch_idle,assume a crouching position with feet almost parallel,assumes a squatting posture with nearly parallel feet in an open position,person is crouching down with their feet in a parallel and open position,A crouched person is standing in place and subtly swaying.,"feet almost parallel in open position, talent crouching",crouch position,crouch idle,neutral,neutral,walking,crouching,0,0,0,0,0,A244,M,169,28,140,29,95,137,173,95,26,50,8,61,27,F,actor
91
+ high_jump_R_opt_1_004__A305,high_jump_R_004__A305,413,Locomotion,Baseline,1.0,False,soma_uniform/bvh/230324/high_jump_R_004__A305.bvh,soma_shapes/soma_base_fit_mhr_params.npz,soma_proportional/bvh/230324/high_jump_R_004__A305.bvh,soma_shapes/soma_proportion_fit_mhr_params/A305.npz,g1/csv/230324/high_jump_R_004__A305.csv,high_jump_R_opt_1_004,A305,high_jump_R_004,230324,_1,high_jump_R_opt_1,high jump over a moving pole in one place,making high jump over a moving pole stationary,performer makes high jump in one place over a moving pole,A person energetically does a high jump while facing forward.,"jump high over moving pole, jump high in one place",jumping high,high jump opt 1,neutral,neutral,jumping,standing,1,1,0,0,0,A305,M,167,25,141,36,87,127,164,97,28,53,10,56,30,F,stuntman
92
+ crouch_idle_right_R_101__A125,idle_crouch_right_loop_101__A125,911,Locomotion,Advanced Locomotion,1.0,False,soma_uniform/bvh/230119/idle_crouch_right_loop_101__A125.bvh,soma_shapes/soma_base_fit_mhr_params.npz,soma_proportional/bvh/230119/idle_crouch_right_loop_101__A125.bvh,soma_shapes/soma_proportion_fit_mhr_params/A125.npz,g1/csv/230119/idle_crouch_right_loop_101__A125.csv,crouch_idle_right_R_101,A125,idle_crouch_right_loop_101,230119,_3,crouch_idle_right_R,character with right foot forward assumes a crouching position leaning on the left foot,stand in a crouching position with right foot forward,maintain a crouching position with the right leg slightly extended forward,A crouched person is standing in place with their right leg ahead of the left.,"right foot slightly forward, body weight on left foot, talent crouching",crouch position right leg forward,crouch idle right,neutral,neutral,crouching idle,crouching,0,0,0,0,0,A125,M,175,26,147,30,96,137,180,102,29,53,8,64,28,F,actor
93
+ walk_ff_start_315_R_101__A174,walk_ff_start_315_101__A174,572,Locomotion,Basic Locomotion Neutral,1.0,False,soma_uniform/bvh/230209/walk_ff_start_315_101__A174.bvh,soma_shapes/soma_base_fit_mhr_params.npz,soma_proportional/bvh/230209/walk_ff_start_315_101__A174.bvh,soma_shapes/soma_proportion_fit_mhr_params/A174.npz,g1/csv/230209/walk_ff_start_315_101__A174.csv,walk_ff_start_315_R_101,A174,walk_ff_start_315_101,230209,_1,walk_ff_start_315_R,walking backward starting with right foot and moving diagonally with left foot crossing behind,"begin by moving right foot, then move backward diagonally crossing with left foot",person takes a right diagonal step backward,A person starts walking swiftly toward the back right diagonal in a crosswise manner at a moderate pace.,"start right foot, move backward diagonally, left foot back crossing, walk backward diagonally",start walking back right diagonal,walk ff start 315,neutral,neutral,walking,standing,1,0,0,0,0,A174,M,169,23,140,29,87,130,163,98,24,51,8,56,31,F,actor
94
+ kneeling_start_R_001__A021,kneeling_start_001__A021,572,Locomotion,Basic Locomotion Neutral,1.0,False,soma_uniform/bvh/220713/kneeling_start_001__A021.bvh,soma_shapes/soma_base_fit_mhr_params.npz,soma_proportional/bvh/220713/kneeling_start_001__A021.bvh,soma_shapes/soma_proportion_fit_mhr_params/A021.npz,g1/csv/220713/kneeling_start_001__A021.csv,kneeling_start_R_001,A021,kneeling_start_001,220713,_1,kneeling_start_R,"person begins to kneel, leading with the right knee","person starts to kneel, putting their right knee down first",person starts to kneel with their right knee first and keeps a straight back,"A person standing upright kneels down while facing forward, and sways slightly.","from standing to kneeling, right knee first, back straight while kneeling",beginning to kneel,kneeling start,neutral,neutral,kneeling,"standing, kneeling",0,0,0,0,0,A021,S,155,23,129,30,85,121,152,85,24,44,7,41,29,F,general
95
+ kneeling_stop_R_001__A021,kneeling_stop_001__A021,389,Locomotion,Basic Locomotion Neutral,1.0,False,soma_uniform/bvh/220713/kneeling_stop_001__A021.bvh,soma_shapes/soma_base_fit_mhr_params.npz,soma_proportional/bvh/220713/kneeling_stop_001__A021.bvh,soma_shapes/soma_proportion_fit_mhr_params/A021.npz,g1/csv/220713/kneeling_stop_001__A021.csv,kneeling_stop_R_001,A021,kneeling_stop_001,220713,_1,kneeling_stop_R,"transition from a kneeling to standing position, leading with the right foot and keeping the back straight","person transitions from kneeling to standing, leading with the right foot and keeping their back straight","individual transitions from a kneeling position to standing upright, leading with their right foot and keeping their back straight",A person kneeling on the ground gets up and stands idle.,"from kneeling to standing, right foot first, back straight while kneeling",stopping kneeling,kneeling stop,neutral,neutral,kneeling,"standing, kneeling",1,1,0,0,0,A021,S,155,23,129,30,85,121,152,85,24,44,7,41,29,F,general
96
+ walk_ff_start_180_R_001__A021,walk_forward_start_001__A021,594,Locomotion,Basic Locomotion Neutral,1.0,False,soma_uniform/bvh/220713/walk_forward_start_001__A021.bvh,soma_shapes/soma_base_fit_mhr_params.npz,soma_proportional/bvh/220713/walk_forward_start_001__A021.bvh,soma_shapes/soma_proportion_fit_mhr_params/A021.npz,g1/csv/220713/walk_forward_start_001__A021.csv,walk_ff_start_180_R_001,A021,walk_forward_start_001,220713,_1,walk_ff_start_180_R,"starting with the right leg, person takes a step forward",initiate a walk forward by stepping out with your right foot,start walking forward beginning with the right foot,A person starts to walk forward at a normal pace.,"start with the right foot, move forward, and walk forward",beginning to walk forward with the right leg,walk ff start 180,neutral,neutral,walking,standing,1,0,0,0,0,A021,S,155,23,129,30,85,121,152,85,24,44,7,41,29,F,general
97
+ walk_ff_loop_270_R_001__A021,walk_sideway_right_loop_001__A021,1801,Locomotion,Basic Locomotion Neutral,1.0,False,soma_uniform/bvh/220713/walk_sideway_right_loop_001__A021.bvh,soma_shapes/soma_base_fit_mhr_params.npz,soma_proportional/bvh/220713/walk_sideway_right_loop_001__A021.bvh,soma_shapes/soma_proportion_fit_mhr_params/A021.npz,g1/csv/220713/walk_sideway_right_loop_001__A021.csv,walk_ff_loop_270_R_001,A021,walk_sideway_right_loop_001,220713,_1,walk_ff_loop_270_R,tired person is taking side steps to the right,tired person walks sideways to the right with lead foot first,tired person walks sideways to the right,"A person is steadily walking sideways in the right direction, taking several steps.","face forward all the time, lead foot first, passing step, walk right, talent tired all the time, low energy",walking sideways to the right,walk ff loop 270,neutral,neutral,walking,standing,1,0,0,0,0,A021,S,155,23,129,30,85,121,152,85,24,44,7,41,29,F,general
98
+ jog_ff_loop_180_R_001__A017,Loop_Forward_Jog_001__A017,14153,Locomotion,Baseline,1.0,False,soma_uniform/bvh/220705/Loop_Forward_Jog_001__A017.bvh,soma_shapes/soma_base_fit_mhr_params.npz,soma_proportional/bvh/220705/Loop_Forward_Jog_001__A017.bvh,soma_shapes/soma_proportion_fit_mhr_params/A017.npz,g1/csv/220705/Loop_Forward_Jog_001__A017.csv,jog_ff_loop_180_R_001,A017,Loop_Forward_Jog_001,220705,_1,jog_ff_loop_180_R,jog forward ,person jogs forward without speeding up to a run,character is progressing in a jogging pace,"A person jogs forward, comes to a halt, stands still, and then tilts to the right.","move forward, jog",jog forward,jog ff loop 180,neutral,neutral,jogging,standing,1,1,0,0,0,A017,T,180,27,148,43,95,144,178,100,27,50,9,60,32,F,general
99
+ walk_ff_loop_180_R_001__A017,Loop_Forward_Walk_001__A017,1013,Locomotion,Basic Locomotion Neutral,1.0,False,soma_uniform/bvh/220705/Loop_Forward_Walk_001__A017.bvh,soma_shapes/soma_base_fit_mhr_params.npz,soma_proportional/bvh/220705/Loop_Forward_Walk_001__A017.bvh,soma_shapes/soma_proportion_fit_mhr_params/A017.npz,g1/csv/220705/Loop_Forward_Walk_001__A017.csv,walk_ff_loop_180_R_001,A017,Loop_Forward_Walk_001,220705,_1,walk_ff_loop_180_R,keep walking forward,character walks forward keeping their face looking front,"someone walks forward, leading with their right foot",A person casually strolls forward at a normal pace.,"consistently face forward, start with the right foot and walk forward",strolling forward,walk ff loop 180,neutral,neutral,walking,standing,1,0,0,0,0,A017,T,180,27,148,43,95,144,178,100,27,50,9,60,32,F,general
100
+ jump_ff_180_R_001__A021,jump_forward_001__A021,1144,Locomotion,Baseline,1.0,False,soma_uniform/bvh/220713/jump_forward_001__A021.bvh,soma_shapes/soma_base_fit_mhr_params.npz,soma_proportional/bvh/220713/jump_forward_001__A021.bvh,soma_shapes/soma_proportion_fit_mhr_params/A021.npz,g1/csv/220713/jump_forward_001__A021.csv,jump_ff_180_R_001,A021,jump_forward_001,220713,_1,jump_ff_180_R,jumping forward from a standing position,make a small forward leap,person makes a small jump as they move forward,"A person jumps forward, pauses briefly in a standing position, then begins to walk backward while turning to their right.","start right foot, move forward, small jump",jump forward,jump ff 180,neutral,neutral,jumping,standing,1,1,0,0,0,A021,S,155,23,129,30,85,121,152,85,24,44,7,41,29,F,general
101
+ jump_ff_180_R_001__A022,jump_forward_001__A022,1260,Locomotion,Baseline,1.0,False,soma_uniform/bvh/220713/jump_forward_001__A022.bvh,soma_shapes/soma_base_fit_mhr_params.npz,soma_proportional/bvh/220713/jump_forward_001__A022.bvh,soma_shapes/soma_proportion_fit_mhr_params/A022.npz,g1/csv/220713/jump_forward_001__A022.csv,jump_ff_180_R_001,A022,jump_forward_001,220713,_1,jump_ff_180_R,jumping forward from a standing position,make a small forward leap,person makes a small jump as they move forward,"A person is jumping forward with their right foot, taking the leap.","start right foot, move forward, small jump",jump forward,jump ff 180,neutral,neutral,jumping,standing,1,1,0,0,0,A022,T,180,25,152,32,93,135,174,105,28,51,7,62,30,F,general
dataset/seed/metadata/seed_metadata_v004_100_move_name.jsonl ADDED
@@ -0,0 +1,100 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {"move_name": "crouch_idle_001__A147"}
2
+ {"move_name": "high_jump_R_opt_1_001__A280"}
3
+ {"move_name": "crouch_idle_right_R_001__A194"}
4
+ {"move_name": "walk_ff_start_315_R_001__A049"}
5
+ {"move_name": "jog_arc_ccw_stop_001__A029"}
6
+ {"move_name": "walk_ff_stop_360_R_001__A017"}
7
+ {"move_name": "crouch_ff_stop_180_R_001__A195"}
8
+ {"move_name": "turn_jump_000_001__A022"}
9
+ {"move_name": "jog_ff_start_180_R_001__A021"}
10
+ {"move_name": "jog_arc_ccw_start_001__A029"}
11
+ {"move_name": "jog_arc_cw_stop_R_001__A049"}
12
+ {"move_name": "walk_ff_stop_180_R_001__A017"}
13
+ {"move_name": "turn_walk_000_L_001__A021"}
14
+ {"move_name": "jog_ff_start_000_001__A025"}
15
+ {"move_name": "jump_ff_000_001__A021"}
16
+ {"move_name": "jog_ff_start_315_R_001__A050"}
17
+ {"move_name": "turn_jump_090_001__A023"}
18
+ {"move_name": "crouch_turn_360_R_001__A126"}
19
+ {"move_name": "turn_walk_045_L_001__A023"}
20
+ {"move_name": "jump_twice_R_001__A533"}
21
+ {"move_name": "relaxed_fast_walk_ff_180_001__A005"}
22
+ {"move_name": "crouch_ff_stop_360_R_001__A126"}
23
+ {"move_name": "high_jump_full_turn_R_opt_1_001__A479"}
24
+ {"move_name": "high_jump_R_opt_2_001__A476"}
25
+ {"move_name": "jump_around_001__A492"}
26
+ {"move_name": "high_jump_ff_180_R_opt_2_001__A476"}
27
+ {"move_name": "high_jump_ff_180_R_opt_1_001__A476"}
28
+ {"move_name": "walk_ff_start_360_R_fast_001__A447"}
29
+ {"move_name": "jog_arc_cw_start_R_001__A093"}
30
+ {"move_name": "jog_ff_start_360_R_normal_pace_001__A451"}
31
+ {"move_name": "jog_arc_cw_start_R_normal_pace_001__A451"}
32
+ {"move_name": "walk_ff_start_360_R_normal_pace_001__A445"}
33
+ {"move_name": "walk_arc_cw_start_R_normal_pace_001__A443"}
34
+ {"move_name": "walk_ff_start_360_R_very_fast_001__A447"}
35
+ {"move_name": "walk_arc_cw_start_R_very_fast_002__A447"}
36
+ {"move_name": "jog_arc_cw_stop_R_fast_001__A452"}
37
+ {"move_name": "walk_arc_cw_stop_R_fast_002__A447"}
38
+ {"move_name": "jog_arc_cw_stop_R_very_fast_001__A455"}
39
+ {"move_name": "walk_ff_stop_360_R_very_fast_002__A447"}
40
+ {"move_name": "touch_toes_standing_R_001__A533"}
41
+ {"move_name": "jump_ff_360_R_002__A045"}
42
+ {"move_name": "crouch_idle_002__A148"}
43
+ {"move_name": "high_jump_R_opt_1_002__A280"}
44
+ {"move_name": "crouch_idle_right_R_002__A194"}
45
+ {"move_name": "walk_ff_start_315_R_002__A049"}
46
+ {"move_name": "jog_arc_ccw_stop_002__A029"}
47
+ {"move_name": "walk_ff_stop_360_R_002__A049"}
48
+ {"move_name": "crouch_ff_stop_180_R_002__A147"}
49
+ {"move_name": "turn_jump_000_002__A022"}
50
+ {"move_name": "jog_ff_start_180_R_002__A021"}
51
+ {"move_name": "jog_arc_ccw_start_002__A029"}
52
+ {"move_name": "jog_arc_cw_stop_R_002__A045"}
53
+ {"move_name": "walk_ff_stop_180_R_002__A023"}
54
+ {"move_name": "turn_walk_000_L_002__A021"}
55
+ {"move_name": "jog_ff_start_000_002__A025"}
56
+ {"move_name": "jump_ff_000_002__A021"}
57
+ {"move_name": "jog_ff_start_315_R_002__A050"}
58
+ {"move_name": "turn_jump_090_002__A025"}
59
+ {"move_name": "crouch_turn_360_R_002__A125"}
60
+ {"move_name": "turn_walk_045_L_002__A023"}
61
+ {"move_name": "jump_twice_R_002__A533"}
62
+ {"move_name": "relaxed_fast_walk_ff_180_002__A005"}
63
+ {"move_name": "crouch_ff_stop_360_R_002__A126"}
64
+ {"move_name": "high_jump_full_turn_R_opt_1_002__A477"}
65
+ {"move_name": "jump_ff_360_R_003__A045"}
66
+ {"move_name": "crouch_idle_003__A148"}
67
+ {"move_name": "high_jump_R_opt_1_003__A280"}
68
+ {"move_name": "crouch_idle_right_R_003__A194"}
69
+ {"move_name": "walk_ff_start_315_R_003__A049"}
70
+ {"move_name": "jog_arc_ccw_stop_003__A029"}
71
+ {"move_name": "walk_ff_stop_360_R_003__A049"}
72
+ {"move_name": "crouch_ff_stop_180_R_003__A146"}
73
+ {"move_name": "turn_jump_000_003__A022"}
74
+ {"move_name": "jog_ff_start_180_R_003__A022"}
75
+ {"move_name": "jog_arc_ccw_start_003__A029"}
76
+ {"move_name": "jog_arc_cw_stop_R_003__A047"}
77
+ {"move_name": "walk_ff_stop_180_R_003__A025"}
78
+ {"move_name": "turn_walk_000_L_003__A021"}
79
+ {"move_name": "jog_ff_start_000_003__A025"}
80
+ {"move_name": "jump_ff_000_003__A021"}
81
+ {"move_name": "jog_ff_start_315_R_003__A050"}
82
+ {"move_name": "turn_jump_090_003__A023"}
83
+ {"move_name": "crouch_turn_360_R_003__A125"}
84
+ {"move_name": "turn_walk_045_L_003__A023"}
85
+ {"move_name": "jump_twice_R_003__A533"}
86
+ {"move_name": "relaxed_fast_walk_ff_180_003__A005"}
87
+ {"move_name": "crouch_ff_stop_360_R_003__A126"}
88
+ {"move_name": "jump_ff_360_R_004__A045"}
89
+ {"move_name": "crouch_idle_004__A244"}
90
+ {"move_name": "high_jump_R_opt_1_004__A305"}
91
+ {"move_name": "crouch_idle_right_R_101__A125"}
92
+ {"move_name": "walk_ff_start_315_R_101__A174"}
93
+ {"move_name": "kneeling_start_R_001__A021"}
94
+ {"move_name": "kneeling_stop_R_001__A021"}
95
+ {"move_name": "walk_ff_start_180_R_001__A021"}
96
+ {"move_name": "walk_ff_loop_270_R_001__A021"}
97
+ {"move_name": "jog_ff_loop_180_R_001__A017"}
98
+ {"move_name": "walk_ff_loop_180_R_001__A017"}
99
+ {"move_name": "jump_ff_180_R_001__A021"}
100
+ {"move_name": "jump_ff_180_R_001__A022"}
flake.lock ADDED
@@ -0,0 +1,27 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "nodes": {
3
+ "nixpkgs": {
4
+ "locked": {
5
+ "lastModified": 1772963539,
6
+ "narHash": "sha256-9jVDGZnvCckTGdYT53d/EfznygLskyLQXYwJLKMPsZs=",
7
+ "owner": "NixOS",
8
+ "repo": "nixpkgs",
9
+ "rev": "9dcb002ca1690658be4a04645215baea8b95f31d",
10
+ "type": "github"
11
+ },
12
+ "original": {
13
+ "owner": "NixOS",
14
+ "ref": "nixos-unstable",
15
+ "repo": "nixpkgs",
16
+ "type": "github"
17
+ }
18
+ },
19
+ "root": {
20
+ "inputs": {
21
+ "nixpkgs": "nixpkgs"
22
+ }
23
+ }
24
+ },
25
+ "root": "root",
26
+ "version": 7
27
+ }
flake.nix ADDED
@@ -0,0 +1,105 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ description = "PSI development shells";
3
+
4
+ inputs = {
5
+ self.submodules = true;
6
+ nixpkgs.url = "github:NixOS/nixpkgs/nixos-unstable";
7
+ };
8
+
9
+ outputs = {
10
+ self,
11
+ nixpkgs,
12
+ }:
13
+ let
14
+ system = "x86_64-linux";
15
+ pkgs = import nixpkgs {
16
+ inherit system;
17
+ config = {
18
+ allowUnfree = true;
19
+ cudaSupport = true;
20
+ };
21
+ };
22
+ lib = pkgs.lib;
23
+ runtimeLib = import ./nix/lib/runtime.nix {
24
+ inherit lib pkgs;
25
+ };
26
+ psiBaseRuntime = import ./nix/runtime-base.nix { inherit pkgs; };
27
+ hostGpuRuntime = import ./nix/runtime-host-gpu.nix { inherit pkgs; };
28
+
29
+ commonPackages = [
30
+ pkgs.bashInteractive
31
+ pkgs.coreutils
32
+ pkgs.curl
33
+ pkgs.git
34
+ pkgs.git-lfs
35
+ pkgs.uv
36
+ pkgs.wget
37
+ ];
38
+
39
+ commonShellHook = ''
40
+ if [ -f "$HOME/.env" ]; then
41
+ set -a
42
+ . "$HOME/.env"
43
+ set +a
44
+ fi
45
+
46
+ _psi_detect_torch_cuda_arch_list() {
47
+ if [ -n "''${TORCH_CUDA_ARCH_LIST:-}" ]; then
48
+ return 0
49
+ fi
50
+
51
+ if ! command -v nvidia-smi >/dev/null 2>&1; then
52
+ return 1
53
+ fi
54
+
55
+ _psi_compute_caps="$(
56
+ nvidia-smi --query-gpu=compute_cap --format=csv,noheader,nounits 2>/dev/null \
57
+ | sed 's/^[[:space:]]*//; s/[[:space:]]*$//' \
58
+ | sed '/^$/d'
59
+ )"
60
+
61
+ if [ -z "$_psi_compute_caps" ]; then
62
+ return 1
63
+ fi
64
+
65
+ _psi_compute_cap_count="$(printf '%s\n' "$_psi_compute_caps" | sort -u | wc -l)"
66
+ if [ "$_psi_compute_cap_count" -ne 1 ]; then
67
+ return 1
68
+ fi
69
+
70
+ _psi_compute_cap="$(printf '%s\n' "$_psi_compute_caps" | head -n 1)"
71
+ if [ -n "$_psi_compute_cap" ]; then
72
+ export TORCH_CUDA_ARCH_LIST="$_psi_compute_cap+PTX"
73
+ return 0
74
+ fi
75
+
76
+ return 1
77
+ }
78
+
79
+ _psi_detect_torch_cuda_arch_list || true
80
+ unset -f _psi_detect_torch_cuda_arch_list
81
+ '';
82
+ in {
83
+ devShells.${system} = {
84
+ default = runtimeLib.mkRuntimeShell {
85
+ name = "psi";
86
+ runtimes = [ psiBaseRuntime hostGpuRuntime ];
87
+ extraPackages = commonPackages;
88
+ extraShellHook = ''
89
+ ${commonShellHook}
90
+ echo "SIMPLE runtime composed into root shell"
91
+ '';
92
+ stdenv = pkgs.gcc13Stdenv;
93
+ };
94
+
95
+ psi = runtimeLib.mkRuntimeShell {
96
+ name = "psi";
97
+ runtimes = [ psiBaseRuntime hostGpuRuntime ];
98
+ extraPackages = commonPackages;
99
+ extraShellHook = commonShellHook;
100
+ stdenv = pkgs.gcc13Stdenv;
101
+ };
102
+
103
+ };
104
+ };
105
+ }
infer.sh ADDED
@@ -0,0 +1,39 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # <modify_textop_repr> infer脚本固定到v4仓库。
2
+ # 原逻辑:
3
+ # cd /cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/Psi0-yth
4
+ cd /cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/Psi0-yth-v18
5
+ export PYTHONPATH=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/Psi0-yth-v18/src
6
+ export CUDA_VISIBLE_DEVICES=0
7
+ pkill -f monitor.py
8
+
9
+ # prompt
10
+ # PYTHONPATH=src /cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/envs/Psi0/bin/python scripts/infer/seed_text_to_motion.py \
11
+ # --run-dir /cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/Psi0-yth/runs/posttrain_v1/posttrain.sg.flow1000.consta.lr1.0e-04.b512.gpus8.2604160750 \
12
+ # --ckpt-step 1300 \
13
+ # --output-dir /cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/Psi0-yth/runs/posttrain_v1/2604160750_v2 \
14
+ # --prompt "walk backward naturally and stop smoothly" \
15
+ # --prompt-motion-name walk_backward_stop \
16
+ # --num-output-frames 400 \
17
+ # --num-inference-steps 10 \
18
+ # --device cuda:0
19
+
20
+ # metadata
21
+ # RUN_DIR="/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/envs/.psi-home/runs/posttrain_v1/posttrain.sg.flow1000.consta.lr1.0e-04.b512.gpus8.2604161308"
22
+ # OUTPUT_DIR="/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/Psi0-yth/runs/posttrain_v1/2604161306"
23
+ # RUN_DIR="/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/envs/.psi-home/runs/posttrain_v1/posttrain.sg.flow1000.consta.lr1.0e-04.b512.gpus8.2604161308"
24
+ # OUTPUT_DIR="/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/Psi0-yth/runs/posttrain_v1/2604161308"
25
+ RUN_DIR="/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/envs/.psi-home/runs/posttrain_v1/posttrain.sg.flow1000.consta.lr1.0e-04.b512.gpus8.2604170218"
26
+ # <modify_states> 默认infer输出改到v2仓库。
27
+ # 原逻辑:
28
+ # OUTPUT_DIR="/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/Psi0-yth/runs/posttrain_v1/2604170218"
29
+ OUTPUT_DIR="/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/Psi0-yth-v18/runs/posttrain_v1/2604170218"
30
+ # RUN_DIR="/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/envs/.psi-home/runs/posttrain_v1/posttrain.sg.flow1000.consta.lr1.0e-04.b512.gpus8.2604170323"
31
+ # OUTPUT_DIR="/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/Psi0-yth/runs/posttrain_v1/2604170323"
32
+ /cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/envs/Psi0/bin/python scripts/infer/seed_text_to_motion.py \
33
+ --run-dir $RUN_DIR \
34
+ --ckpt-step 55000 \
35
+ --output-dir $OUTPUT_DIR \
36
+ --num-samples 1 \
37
+ --start-index 8 \
38
+ --num-inference-steps 10 \
39
+ --device cuda:0
infer_prompt.sh ADDED
@@ -0,0 +1,213 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ cd /cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/Psi0-yth-v18
2
+ source /cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/Psi0-yth-v15/.venv-psi/bin/activate
3
+ [ -f .env ] && source .env
4
+
5
+ export PYTHONPATH=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/Psi0-yth-v18/src
6
+ export CUDA_VISIBLE_DEVICES=0
7
+ export PSI_INFER_FLOAT32=1
8
+ pkill -f monitor.py
9
+
10
+ # 改这里选择推理类型:audio | audio_command | audio_command_only | text | video | video_clip
11
+ MODE=video_clip
12
+
13
+ NUM_INFERENCE_STEPS=10
14
+ DEVICE=cuda:0
15
+
16
+ RUN_TIMESTAMP=2606041200
17
+ # RUN_TIMESTAMP=2606041447
18
+ RUN_PARENT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/Psi0-yth-v18/runs/posttrain_v1_trimodal
19
+ RUN_DIR=$RUN_PARENT/posttrain.sg.flow1000.cwml.lr1.0e-04.b256.gpus8.$RUN_TIMESTAMP
20
+ CKPT_STEP=best
21
+
22
+ # 普通 audio / video 默认样例
23
+ MOTION_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/GR00T-WholeBodyControl/gear_sonic_deploy/reference/converted/psi0_dance/gvhmr
24
+ VIDEO_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/dance_1723_video_psi0/dance_1723_video_10s
25
+ MOTION_NAME=movies__F-SNBR4jhY_0007
26
+ # MOTION_DIR=$MOTION_ROOT/$MOTION_NAME
27
+ # VIDEO_PATH=$VIDEO_ROOT/$MOTION_NAME.mp4
28
+ # VIDEO_PATH=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/videogen/motion_example_0506_demo/original_videos/00000/0h7AlrKLslw_00056_0_313.mp4
29
+ # VIDEO_PATH=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/dance_1723_video_psi0/dance_1723_video_10s/movies__F-SNBR4jhY_0007.mp4
30
+
31
+ MOTION_DIR=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/GR00T-WholeBodyControl/gear_sonic_deploy/reference/converted/psi0_dance/gvhmr/movies__6RU6bQLqVk_0000
32
+ VIDEO_PATH=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/Psi0-yth-v18/kun3.mp4
33
+
34
+ # MOTION_DIR=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/videogen/psi0_video_clip_50f_rand8/dance_1723_video_10s/motion/movies__F-SNBR4jhY_0007.chunk00.f000160
35
+ # VIDEO_PATH=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/videogen/psi0_video_clip_50f_rand8/dance_1723_video_10s/videos/movies__F-SNBR4jhY_0007.chunk00.f000160.mp4
36
+
37
+ # MOTION_DIR=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/videogen/psi0_video_clip_50f_rand32_full/motion_example_0506_demo/motion/converted_motion__00000___dFAYYMJbWk_00193_268_382.chunk00.f000000
38
+ # VIDEO_PATH=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/videogen/psi0_video_clip_50f_rand32_full/motion_example_0506_demo/videos/converted_motion__00000___dFAYYMJbWk_00193_268_382.chunk00.f000000.mp4
39
+
40
+ # audio_command 默认样例;AUDIO_COMMAND_AUDIO_PATH 是目录时会随机抽一条 wav
41
+ # AUDIO_COMMAND_MOTION_DIR=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/Psi0/dataset/seed_all/g1/csv_sonic/230530/sitting_legs_bend_arms_front_loop_R_001__A410
42
+ # AUDIO_COMMAND_AUDIO_PATH=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/qwen3-tts/seed_audio/003856
43
+ # AUDIO_COMMAND_AUDIO_PATH=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/qwen3-tts/seed_audio/000017
44
+ AUDIO_COMMAND_MOTION_DIR=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/GR00T-WholeBodyControl/gear_sonic_deploy/reference/converted/psi0_dance/gvhmr/movies_1mzl0Zh614A_0000
45
+ AUDIO_COMMAND_PROMPT="Generate a motion that follows the spoken instruction in the audio"
46
+ AUDIO_COMMAND_AUDIO_PATH=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/Psi0-yth-v18/jump.wav
47
+
48
+ # text 默认样例
49
+ TEXT_PROMPT="Generate the following motion sequence: A person jog forward."
50
+ TEXT_START_INDEX=0
51
+ TEXT_NUM_OUTPUT_FRAMES=300
52
+ PROMPT_MOTION_NAME=text_prompt
53
+
54
+ case "$MODE" in
55
+ audio|audio_text|audio+text|audio_only|audio_pure)
56
+ MODE_NAME=audio
57
+ AUDIO_CONFIG_KEY=audio
58
+ OUTPUT_DIR=$RUN_PARENT/$RUN_TIMESTAMP.$MODE_NAME.test
59
+ PROMPT="Generate a dance sequence that matches the provided music."
60
+ if [ "$MODE" = "audio_only" ] || [ "$MODE" = "audio_pure" ]; then
61
+ PROMPT=""
62
+ fi
63
+
64
+ # rm -rf "$OUTPUT_DIR"
65
+
66
+ echo "Mode: $MODE_NAME"
67
+ echo "Audio config key: $AUDIO_CONFIG_KEY"
68
+ echo "Run dir: $RUN_DIR"
69
+ echo "Ckpt step: $CKPT_STEP"
70
+ echo "Prompt: $PROMPT"
71
+ echo "Motion dir: $MOTION_DIR"
72
+ echo "Video path: $VIDEO_PATH"
73
+ echo "Output dir: $OUTPUT_DIR"
74
+
75
+ python scripts/infer/seed_audio_to_motion.py \
76
+ --run-dir "$RUN_DIR" \
77
+ --ckpt-step "$CKPT_STEP" \
78
+ --output-dir "$OUTPUT_DIR" \
79
+ --motion-dir "$MOTION_DIR" \
80
+ --video-path "$VIDEO_PATH" \
81
+ --audio-config-key "$AUDIO_CONFIG_KEY" \
82
+ --prompt "$PROMPT" \
83
+ --num-inference-steps "$NUM_INFERENCE_STEPS" \
84
+ --device "$DEVICE" || return 1 2>/dev/null || exit 1
85
+ ;;
86
+ audio_command|audio_command_text|audio_command_only|command_audio)
87
+ MODE_NAME=audio_command
88
+ AUDIO_CONFIG_KEY=audio_command
89
+ OUTPUT_DIR=$RUN_PARENT/$RUN_TIMESTAMP.$MODE_NAME.test
90
+ MOTION_DIR=$AUDIO_COMMAND_MOTION_DIR
91
+ VIDEO_PATH=$AUDIO_COMMAND_AUDIO_PATH
92
+ PROMPT=$AUDIO_COMMAND_PROMPT
93
+ if [ "$MODE" = "audio_command_only" ]; then
94
+ PROMPT=""
95
+ fi
96
+
97
+ # rm -rf "$OUTPUT_DIR"
98
+
99
+ echo "Mode: $MODE_NAME"
100
+ echo "Audio config key: $AUDIO_CONFIG_KEY"
101
+ echo "Run dir: $RUN_DIR"
102
+ echo "Ckpt step: $CKPT_STEP"
103
+ echo "Prompt: $PROMPT"
104
+ echo "Motion dir: $MOTION_DIR"
105
+ echo "Video path: $VIDEO_PATH"
106
+ echo "Output dir: $OUTPUT_DIR"
107
+
108
+ python scripts/infer/seed_audio_to_motion.py \
109
+ --run-dir "$RUN_DIR" \
110
+ --ckpt-step "$CKPT_STEP" \
111
+ --output-dir "$OUTPUT_DIR" \
112
+ --motion-dir "$MOTION_DIR" \
113
+ --video-path "$VIDEO_PATH" \
114
+ --audio-config-key "$AUDIO_CONFIG_KEY" \
115
+ --prompt "$PROMPT" \
116
+ --num-inference-steps "$NUM_INFERENCE_STEPS" \
117
+ --device "$DEVICE" || return 1 2>/dev/null || exit 1
118
+ ;;
119
+ text|pure_text)
120
+ MODE_NAME=text
121
+ OUTPUT_DIR=$RUN_PARENT/$RUN_TIMESTAMP.$MODE_NAME.test
122
+
123
+ # rm -rf "$OUTPUT_DIR"
124
+
125
+ echo "Mode: $MODE_NAME"
126
+ echo "Run dir: $RUN_DIR"
127
+ echo "Ckpt step: $CKPT_STEP"
128
+ echo "Prompt: $TEXT_PROMPT"
129
+ echo "Prompt motion name: $PROMPT_MOTION_NAME"
130
+ echo "Start index: $TEXT_START_INDEX"
131
+ echo "Num output frames: $TEXT_NUM_OUTPUT_FRAMES"
132
+ echo "Output dir: $OUTPUT_DIR"
133
+
134
+ python scripts/infer/seed_text_to_motion.py \
135
+ --run-dir "$RUN_DIR" \
136
+ --ckpt-step "$CKPT_STEP" \
137
+ --output-dir "$OUTPUT_DIR" \
138
+ --prompt "$TEXT_PROMPT" \
139
+ --prompt-motion-name "$PROMPT_MOTION_NAME" \
140
+ --init-motion-dir "$MOTION_DIR" \
141
+ --start-index "$TEXT_START_INDEX" \
142
+ --num-output-frames "$TEXT_NUM_OUTPUT_FRAMES" \
143
+ --num-inference-steps "$NUM_INFERENCE_STEPS" \
144
+ --force-text-only \
145
+ --device "$DEVICE" || return 1 2>/dev/null || exit 1
146
+ ;;
147
+ video|video_text|video+text|video_only|video_pure|video_clip|video_clip_text|video_clip_only)
148
+ if [ "$MODE" = "video_clip" ] || [ "$MODE" = "video_clip_text" ] || [ "$MODE" = "video_clip_only" ]; then
149
+ MODE_NAME=video_clip
150
+ VIDEO_CONFIG_KEY=video_clip
151
+ VIDEO_CLIP_ARGS=(--video-config-key video_clip --video-clip)
152
+ else
153
+ MODE_NAME=video
154
+ VIDEO_CONFIG_KEY=video
155
+ VIDEO_CLIP_ARGS=(--video-config-key video)
156
+ fi
157
+ OUTPUT_DIR=$RUN_PARENT/$RUN_TIMESTAMP.$MODE_NAME.test
158
+ PROMPT="Generate a motion sequence that matches the provided video."
159
+ if [ "$MODE" = "video_only" ] || [ "$MODE" = "video_pure" ] || [ "$MODE" = "video_clip_only" ]; then
160
+ PROMPT=""
161
+ fi
162
+
163
+ # rm -rf "$OUTPUT_DIR"
164
+
165
+ echo "Mode: $MODE_NAME"
166
+ echo "Video config key: $VIDEO_CONFIG_KEY"
167
+ echo "Run dir: $RUN_DIR"
168
+ echo "Ckpt step: $CKPT_STEP"
169
+ echo "Prompt: $PROMPT"
170
+ echo "Motion dir: $MOTION_DIR"
171
+ echo "Video path: $VIDEO_PATH"
172
+ echo "Output dir: $OUTPUT_DIR"
173
+
174
+ python scripts/infer/seed_video_to_motion.py \
175
+ --run-dir "$RUN_DIR" \
176
+ --ckpt-step "$CKPT_STEP" \
177
+ --output-dir "$OUTPUT_DIR" \
178
+ --motion-dir "$MOTION_DIR" \
179
+ --video-path "$VIDEO_PATH" \
180
+ --prompt "$PROMPT" \
181
+ --num-inference-steps "$NUM_INFERENCE_STEPS" \
182
+ "${VIDEO_CLIP_ARGS[@]}" \
183
+ --device "$DEVICE" || return 1 2>/dev/null || exit 1
184
+ ;;
185
+ *)
186
+ echo "ERROR: unknown mode '$MODE'. Use 'audio', 'audio_command', 'audio_command_only', 'text', 'video', or 'video_clip'."
187
+ exit 1
188
+ ;;
189
+ esac
190
+
191
+ deactivate
192
+ unset PYTHONPATH
193
+ cd /cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/GR00T-WholeBodyControl/gear_sonic
194
+ export PYOPENGL_PLATFORM="egl"
195
+ export MUJOCO_GL="egl"
196
+ export QT_X11_NO_MITSHM=1
197
+ xvfb-run -a -s "-screen 0 1280x1024x24 -extension MIT-SHM" /cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/envs/gmr/bin/python scripts/batch_render_reference_motion_mujoco.py \
198
+ --motion-dir "$OUTPUT_DIR" \
199
+ --source-fps 50 \
200
+ --target-fps 30 \
201
+ --dof-order isaaclab
202
+
203
+ # deactivate
204
+ # unset PYTHONPATH
205
+ # cd /cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/GR00T-WholeBodyControl/gear_sonic
206
+ # export PYOPENGL_PLATFORM="egl"
207
+ # export MUJOCO_GL="egl"
208
+ # export QT_X11_NO_MITSHM=1
209
+ # xvfb-run -a -s "-screen 0 1280x1024x24 -extension MIT-SHM" /cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/envs/gmr/bin/python scripts/batch_render_reference_motion_mujoco.py \
210
+ # --motion-dir "/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/videogen/psi0_video_clip_50f_rand32_full/motion_example_0506_demo/motion/converted_motion__00000___dFAYYMJbWk_00193_268_382.chunk00.f000000" \
211
+ # --source-fps 50 \
212
+ # --target-fps 30 \
213
+ # --dof-order isaaclab
infer_prompt_distill.sh ADDED
@@ -0,0 +1,104 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/usr/bin/env bash
2
+ pkill -f monitor.py
3
+
4
+ ############################
5
+ # Edit config only here.
6
+ ############################
7
+
8
+ STAGE=student
9
+
10
+ # RUN_DIR=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/Psi0-yth-v18/runs/posttrain_v1_motion_distill/posttrain.motion_distill_teacher.sg.flow1000.cwml.lr1.0e-04.b512.gpus8.2605302159
11
+ RUN_DIR=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/Psi0-yth-v18/runs/posttrain_v1_motion_distill/posttrain.motion_distill_student.sg.flow1000.cwml.lr1.0e-04.b256.gpus8.2605310757
12
+ CKPT_STEP=best
13
+ # OUTPUT_DIR=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/Psi0-yth-v18/runs/posttrain_v1_motion_distill/teacher_reconstruct.test
14
+ OUTPUT_DIR=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/Psi0-yth-v18/runs/posttrain_v1_motion_distill/student_infer.test
15
+
16
+ MOTION_DIR=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/videogen/psi0_video_clip_50f_rand8/dance_1723_video_10s/motion/movies__F-SNBR4jhY_0007.chunk00.f000160
17
+ VIDEO_PATH=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/videogen/psi0_video_clip_50f_rand8/dance_1723_video_10s/videos/movies__F-SNBR4jhY_0007.chunk00.f000160.mp4
18
+ PROMPT="Generate a motion sequence that matches the provided video."
19
+
20
+ NUM_OUTPUT_FRAMES=0
21
+ NUM_INFERENCE_STEPS=10
22
+ DEVICE=cuda:0
23
+ CUDA_VISIBLE_DEVICES=0
24
+ PSI_INFER_FLOAT32=1
25
+
26
+ VIDEO_CONFIG_KEY=video_clip
27
+ VIDEO_CLIP=1
28
+ RENDER=1
29
+
30
+ REPO_DIR=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/Psi0-yth-v18
31
+ GEAR_SONIC_DIR=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/GR00T-WholeBodyControl/gear_sonic
32
+ GMR_PYTHON=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/envs/gmr/bin/python
33
+
34
+ ############################
35
+
36
+ cd "$REPO_DIR"
37
+
38
+ source .venv-psi/bin/activate
39
+ [ -f .env ] && source .env
40
+
41
+ export PYTHONPATH=$REPO_DIR/src
42
+ export CUDA_VISIBLE_DEVICES
43
+ export PSI_INFER_FLOAT32
44
+
45
+ pkill -f monitor.py 2>/dev/null || true
46
+ rm -rf "$OUTPUT_DIR"
47
+
48
+ echo "Stage: $STAGE"
49
+ echo "Run dir: $RUN_DIR"
50
+ echo "Ckpt step: $CKPT_STEP"
51
+ echo "Motion dir: $MOTION_DIR"
52
+ echo "Video path: $VIDEO_PATH"
53
+ echo "Output dir: $OUTPUT_DIR"
54
+ echo "Num output frames: $NUM_OUTPUT_FRAMES"
55
+ echo "Num inference steps: $NUM_INFERENCE_STEPS"
56
+ echo "Device: $DEVICE"
57
+
58
+ if [ "$STAGE" = "teacher" ]; then
59
+ python scripts/infer/seed_motion_teacher_reconstruct.py \
60
+ --run-dir "$RUN_DIR" \
61
+ --ckpt-step "$CKPT_STEP" \
62
+ --output-dir "$OUTPUT_DIR" \
63
+ --motion-dir "$MOTION_DIR" \
64
+ --num-output-frames "$NUM_OUTPUT_FRAMES" \
65
+ --num-inference-steps "$NUM_INFERENCE_STEPS" \
66
+ --device "$DEVICE"
67
+ elif [ "$STAGE" = "student" ]; then
68
+ VIDEO_ARGS=(--video-config-key "$VIDEO_CONFIG_KEY")
69
+ if [ "$VIDEO_CLIP" = "1" ]; then
70
+ VIDEO_ARGS+=(--video-clip)
71
+ fi
72
+
73
+ python scripts/infer/seed_video_to_motion.py \
74
+ --run-dir "$RUN_DIR" \
75
+ --ckpt-step "$CKPT_STEP" \
76
+ --output-dir "$OUTPUT_DIR" \
77
+ --motion-dir "$MOTION_DIR" \
78
+ --video-path "$VIDEO_PATH" \
79
+ --prompt "$PROMPT" \
80
+ --num-output-frames "$NUM_OUTPUT_FRAMES" \
81
+ --num-inference-steps "$NUM_INFERENCE_STEPS" \
82
+ "${VIDEO_ARGS[@]}" \
83
+ --device "$DEVICE"
84
+ else
85
+ echo "ERROR: STAGE must be teacher or student."
86
+ exit 1
87
+ fi
88
+
89
+ deactivate
90
+ unset PYTHONPATH
91
+
92
+ if [ "$RENDER" != "1" ]; then
93
+ exit 0
94
+ fi
95
+
96
+ cd "$GEAR_SONIC_DIR"
97
+ export PYOPENGL_PLATFORM=egl
98
+ export MUJOCO_GL=egl
99
+ export QT_X11_NO_MITSHM=1
100
+ xvfb-run -a -s "-screen 0 1280x1024x24 -extension MIT-SHM" "$GMR_PYTHON" scripts/batch_render_reference_motion_mujoco.py \
101
+ --motion-dir "$OUTPUT_DIR" \
102
+ --source-fps 50 \
103
+ --target-fps 30 \
104
+ --dof-order isaaclab
infer_prompt_short.sh ADDED
@@ -0,0 +1,157 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/usr/bin/env bash
2
+ set -euo pipefail
3
+
4
+ cd /cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/Psi0-yth-v18
5
+ source .venv-psi/bin/activate
6
+ [ -f .env ] && source .env
7
+
8
+ export PYTHONPATH=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/Psi0-yth-v18/src
9
+ export CUDA_VISIBLE_DEVICES=0
10
+ export PSI_INFER_FLOAT32=1
11
+ pkill -f monitor.py || true
12
+
13
+ # 改这里:video2motion 只 rollout 多少个实际执行 chunk。
14
+ SHORT_NUM_CHUNKS=2
15
+ # 改这里选择短推理类型:video | video_only | video_clip | video_clip_only
16
+ MODE=video_clip
17
+
18
+ NUM_INFERENCE_STEPS=10
19
+ DEVICE=cuda:0
20
+
21
+ RUN_TIMESTAMP=2605211455
22
+ RUN_PARENT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/Psi0-yth-v18/runs/posttrain_v1_trimodal
23
+ RUN_DIR=$RUN_PARENT/posttrain.sg.flow1000.cwml.lr1.0e-04.b512.gpus8.$RUN_TIMESTAMP
24
+ CKPT_STEP=best
25
+
26
+ MOTION_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/GR00T-WholeBodyControl/gear_sonic_deploy/reference/converted/psi0_dance/gvhmr
27
+ VIDEO_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/dance_1723_video_psi0/dance_1723_video_10s
28
+ MOTION_NAME=movies_1mzl0Zh614A_0000
29
+ MOTION_DIR=$MOTION_ROOT/$MOTION_NAME
30
+ # VIDEO_PATH=$VIDEO_ROOT/$MOTION_NAME.mp4
31
+ # VIDEO_PATH=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/videogen/motion_example_0506_demo/original_videos/00000/0h7AlrKLslw_00056_0_313.mp4
32
+ VIDEO_PATH=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/dance_1723_video_psi0/dance_1723_video_10s/movies__F-SNBR4jhY_0007.mp4
33
+
34
+ PROMPT="Generate a motion sequence that matches the provided video."
35
+ # PROMPT=""
36
+ TARGET_FPS=50
37
+
38
+ if [ "$SHORT_NUM_CHUNKS" -le 0 ]; then
39
+ echo "ERROR: SHORT_NUM_CHUNKS must be positive, got $SHORT_NUM_CHUNKS"
40
+ exit 1
41
+ fi
42
+
43
+ case "$MODE" in
44
+ video|video_text|video+text|video_only|video_pure)
45
+ MODE_NAME=video
46
+ VIDEO_CONFIG_KEY=video
47
+ VIDEO_CLIP_ARGS=(--video-config-key video)
48
+ ;;
49
+ video_clip|video_clip_text|video_clip_only)
50
+ MODE_NAME=video_clip
51
+ VIDEO_CONFIG_KEY=video_clip
52
+ VIDEO_CLIP_ARGS=(--video-config-key video_clip --video-clip --video-clip-motion-fps "$TARGET_FPS")
53
+ ;;
54
+ *)
55
+ echo "ERROR: unknown mode '$MODE'. Use 'video' or 'video_clip'."
56
+ exit 1
57
+ ;;
58
+ esac
59
+
60
+ if [ "$MODE" = "video_only" ] || [ "$MODE" = "video_pure" ] || [ "$MODE" = "video_clip_only" ]; then
61
+ PROMPT=""
62
+ fi
63
+
64
+ RUN_CONFIG=$RUN_DIR/run_config.json
65
+ if [ ! -f "$RUN_CONFIG" ]; then
66
+ echo "ERROR: run_config.json not found: $RUN_CONFIG"
67
+ exit 1
68
+ fi
69
+
70
+ read -r RTC_ENABLED ACTION_HORIZON EXEC_HORIZON <<< "$(python -c 'import json, sys
71
+ cfg = json.load(open(sys.argv[1], encoding="utf-8"))
72
+ model = cfg.get("model", {})
73
+ rtc = bool(model.get("rtc", False))
74
+ action_horizon = int(model.get("action_chunk_size", 30))
75
+ if rtc:
76
+ exec_horizon = int(model.get("action_exec_horizon", action_horizon))
77
+ else:
78
+ exec_horizon = action_horizon
79
+ print(int(rtc), action_horizon, exec_horizon)' "$RUN_CONFIG")"
80
+ NUM_OUTPUT_FRAMES=$((SHORT_NUM_CHUNKS * EXEC_HORIZON))
81
+ if [ "$MODE_NAME" = "video_clip" ]; then
82
+ VIDEO_INPUT_FRAMES=$(((SHORT_NUM_CHUNKS - 1) * EXEC_HORIZON + ACTION_HORIZON))
83
+ else
84
+ VIDEO_INPUT_FRAMES=$NUM_OUTPUT_FRAMES
85
+ fi
86
+ OUTPUT_DIR=$RUN_PARENT/$RUN_TIMESTAMP.$MODE_NAME.short_${SHORT_NUM_CHUNKS}chunks
87
+ VIDEO_DURATION_SEC=$(python -c 'import sys
88
+ frames = int(sys.argv[1])
89
+ fps = float(sys.argv[2])
90
+ print(f"{frames / fps:.6f}")' "$VIDEO_INPUT_FRAMES" "$TARGET_FPS")
91
+ SOURCE_VIDEO_DURATION_SEC=$(ffprobe -v error -show_entries format=duration -of default=nk=1:nw=1 "$VIDEO_PATH")
92
+ python -c 'import sys
93
+ need = float(sys.argv[1])
94
+ have = float(sys.argv[2])
95
+ if have + 1e-3 < need:
96
+ raise SystemExit(f"ERROR: source video is shorter than requested clip: need {need:.6f}s, have {have:.6f}s")' \
97
+ "$VIDEO_DURATION_SEC" "$SOURCE_VIDEO_DURATION_SEC"
98
+ SHORT_INPUT_DIR=$RUN_PARENT/$RUN_TIMESTAMP.$MODE_NAME.short_inputs
99
+ VIDEO_BASENAME=$(basename "$VIDEO_PATH")
100
+ SHORT_VIDEO_PATH=$SHORT_INPUT_DIR/${VIDEO_BASENAME%.*}.short_${SHORT_NUM_CHUNKS}chunks.mp4
101
+
102
+ rm -rf "$OUTPUT_DIR"
103
+ mkdir -p "$SHORT_INPUT_DIR"
104
+ ffmpeg -hide_banner -loglevel error -y \
105
+ -ss 0 \
106
+ -t "$VIDEO_DURATION_SEC" \
107
+ -i "$VIDEO_PATH" \
108
+ -map 0:v:0 \
109
+ -an \
110
+ -c:v libx264 \
111
+ -pix_fmt yuv420p \
112
+ "$SHORT_VIDEO_PATH"
113
+ CROPPED_VIDEO_DURATION_SEC=$(ffprobe -v error -show_entries format=duration -of default=nk=1:nw=1 "$SHORT_VIDEO_PATH")
114
+
115
+ echo "Mode: $MODE_NAME short"
116
+ echo "Video config key: $VIDEO_CONFIG_KEY"
117
+ echo "Short chunks: $SHORT_NUM_CHUNKS"
118
+ echo "RTC enabled: $RTC_ENABLED"
119
+ echo "Action horizon: $ACTION_HORIZON"
120
+ echo "RTC overlap frames: $((ACTION_HORIZON - EXEC_HORIZON))"
121
+ echo "Exec horizon: $EXEC_HORIZON"
122
+ echo "Num output frames: $NUM_OUTPUT_FRAMES"
123
+ echo "Video input frames: $VIDEO_INPUT_FRAMES"
124
+ echo "Target fps: $TARGET_FPS"
125
+ echo "Short video seconds: $VIDEO_DURATION_SEC"
126
+ echo "Cropped video seconds: $CROPPED_VIDEO_DURATION_SEC"
127
+ echo "Run dir: $RUN_DIR"
128
+ echo "Ckpt step: $CKPT_STEP"
129
+ echo "Prompt: $PROMPT"
130
+ echo "Motion dir: $MOTION_DIR"
131
+ echo "Video path: $VIDEO_PATH"
132
+ echo "Short video path: $SHORT_VIDEO_PATH"
133
+ echo "Output dir: $OUTPUT_DIR"
134
+
135
+ python scripts/infer/seed_video_to_motion.py \
136
+ --run-dir "$RUN_DIR" \
137
+ --ckpt-step "$CKPT_STEP" \
138
+ --output-dir "$OUTPUT_DIR" \
139
+ --motion-dir "$MOTION_DIR" \
140
+ --video-path "$SHORT_VIDEO_PATH" \
141
+ --prompt "$PROMPT" \
142
+ --num-output-frames "$NUM_OUTPUT_FRAMES" \
143
+ --num-inference-steps "$NUM_INFERENCE_STEPS" \
144
+ "${VIDEO_CLIP_ARGS[@]}" \
145
+ --device "$DEVICE"
146
+
147
+ deactivate
148
+ unset PYTHONPATH
149
+ cd /cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/GR00T-WholeBodyControl/gear_sonic
150
+ export PYOPENGL_PLATFORM="egl"
151
+ export MUJOCO_GL="egl"
152
+ export QT_X11_NO_MITSHM=1
153
+ xvfb-run -a -s "-screen 0 1280x1024x24 -extension MIT-SHM" /cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/envs/gmr/bin/python scripts/batch_render_reference_motion_mujoco.py \
154
+ --motion-dir "$OUTPUT_DIR" \
155
+ --source-fps 50 \
156
+ --target-fps 30 \
157
+ --dof-order isaaclab
infer_render.sh ADDED
@@ -0,0 +1,64 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ cd /cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/Psi0-yth-v18
2
+ source .venv-psi/bin/activate
3
+ [ -f .env ] && source .env
4
+
5
+ export PYTHONPATH=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/Psi0-yth-v18/src
6
+ export CUDA_VISIBLE_DEVICES=0
7
+ export PSI_INFER_FLOAT32=1
8
+ pkill -f monitor.py
9
+
10
+ # <modify_audio_infer> 推理脚本切换为音频输入,对齐当前audio-only + full-audio + 39D TextOp训练逻辑。
11
+ # 原逻辑:
12
+ # RUN_DIR="/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/Psi0-yth-v18/runs/posttrain_v1/posttrain.sg.flow1000.cwml.lr1.0e-04.b2048.gpus8.2604250959"
13
+ # OUTPUT_DIR="/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/Psi0-yth-v18/runs/posttrain_v1/2604250959"
14
+ # /cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/envs/Psi0/bin/python scripts/infer/seed_text_to_motion.py \
15
+ # --run-dir $RUN_DIR \
16
+ # --ckpt-step best \
17
+ # --output-dir $OUTPUT_DIR \
18
+ # --num-samples 1 \
19
+ # --start-index 23069 \
20
+ # --num-inference-steps 10 \
21
+ # --device cuda:0
22
+
23
+ RUN_TIMESTAMP=2604250426
24
+ RUN_PARENT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/Psi0-yth-v18/runs/posttrain_v1_audio_dance
25
+ RUN_DIR=$RUN_PARENT/posttrain.sag.flow1000.cwml.lr1.0e-04.b512.gpus8.$RUN_TIMESTAMP
26
+ OUTPUT_DIR=$RUN_PARENT/$RUN_TIMESTAMP.best
27
+ MOTION_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/GR00T-WholeBodyControl/gear_sonic_deploy/reference/converted/psi0_dance/gvhmr
28
+ VIDEO_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/dance_1723_video_psi0/dance_1723_video_10s
29
+ MOTION_NAME=movies_1mzl0Zh614A_0000
30
+ MOTION_DIR=$MOTION_ROOT/$MOTION_NAME
31
+ VIDEO_PATH=$VIDEO_ROOT/$MOTION_NAME.mp4
32
+ CKPT_STEP=best
33
+ PROMPT="Generate a dance sequence that matches the provided music."
34
+
35
+ rm -rf $OUTPUT_DIR
36
+
37
+ echo "Run dir: $RUN_DIR"
38
+ echo "Ckpt step: $CKPT_STEP"
39
+ echo "Prompt: $PROMPT"
40
+ echo "Motion dir: $MOTION_DIR"
41
+ echo "Video path: $VIDEO_PATH"
42
+ echo "Output dir: $OUTPUT_DIR"
43
+
44
+ python scripts/infer/seed_audio_to_motion.py \
45
+ --run-dir "$RUN_DIR" \
46
+ --ckpt-step "$CKPT_STEP" \
47
+ --output-dir "$OUTPUT_DIR" \
48
+ --motion-dir "$MOTION_DIR" \
49
+ --video-path "$VIDEO_PATH" \
50
+ --prompt "$PROMPT" \
51
+ --num-inference-steps 10 \
52
+ --device cuda:0
53
+
54
+ deactivate
55
+ unset PYTHONPATH
56
+ cd /cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/GR00T-WholeBodyControl/gear_sonic
57
+ conda activate /cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/envs/gmr
58
+ export PYOPENGL_PLATFORM="egl"
59
+ export MUJOCO_GL="egl"
60
+ xvfb-run -a python scripts/batch_render_reference_motion_mujoco.py \
61
+ --motion-dir "$OUTPUT_DIR" \
62
+ --source-fps 50 \
63
+ --target-fps 30 \
64
+ --dof-order isaaclab
infer_render_auto.sh ADDED
@@ -0,0 +1,711 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/bin/bash
2
+
3
+ REPO_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/Psi0-yth-v21
4
+ DEFAULT_RUN_DIR=$REPO_ROOT/runs/posttrain_v1_trimodal/posttrain_actionhead_M.sg.flow1000.cwml.lr1.0e-04.b256.gpus8.2606241549
5
+
6
+ cd "$REPO_ROOT"
7
+ export PYTHONPATH="$REPO_ROOT/src"
8
+
9
+ source /cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/Psi0-yth-v15/.venv-psi/bin/activate
10
+ [ -f .env ] && source .env
11
+
12
+ export PYTHONPATH="$REPO_ROOT/src"
13
+ export CUDA_VISIBLE_DEVICES=${CUDA_VISIBLE_DEVICES:-0}
14
+ export PSI_INFER_FLOAT32=1
15
+ pkill -f monitor.py || true
16
+
17
+ canonical_mode() {
18
+ case "$1" in
19
+ all|multi|full)
20
+ echo all
21
+ ;;
22
+ text|pure_text)
23
+ echo text
24
+ ;;
25
+ audio|audio_text|audio_only|audio_pure)
26
+ echo audio
27
+ ;;
28
+ audio_command|audio_command_text|audio_command_only|command_audio)
29
+ echo audio_command
30
+ ;;
31
+ video_clip|video_clip_text|video_clip_only)
32
+ echo video_clip
33
+ ;;
34
+ video|video_text|video_only)
35
+ echo video
36
+ ;;
37
+ *)
38
+ echo "$1"
39
+ ;;
40
+ esac
41
+ }
42
+
43
+ RAW_MODE=${PSI_AUTO_INFER_MODE:-all}
44
+ MODE=$(canonical_mode "$RAW_MODE")
45
+ MODE_SEQUENCE=(text audio audio_command video_clip)
46
+ # MODE_SEQUENCE=(audio_command)
47
+
48
+ RUNS_ROOT=${RUNS_ROOT:-$REPO_ROOT/runs/posttrain_v1_trimodal}
49
+ INFER_PY=${INFER_PY:-/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/Psi0-yth-v15/.venv-psi/bin/python}
50
+ # RUN_DIR=${RUN_DIR:-auto}
51
+ RUN_DIR=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/Psi0-yth-v21/runs/posttrain_v1_trimodal/posttrain.sg.flow1000.cwml.lr1.0e-04.b512.gpus8.2607051648
52
+
53
+ if [ $# -ge 1 ]; then
54
+ RUN_DIR=$1
55
+ elif [ "$RUN_DIR" = "auto" ] || [ -z "$RUN_DIR" ]; then
56
+ AUTO_RUN_DIR=$("$INFER_PY" - "$MODE" "$RUNS_ROOT" <<'PY'
57
+ import json
58
+ import re
59
+ import sys
60
+ from pathlib import Path
61
+
62
+ mode = sys.argv[1]
63
+ runs_root = Path(sys.argv[2])
64
+
65
+ def ratio(data, key):
66
+ try:
67
+ return float(data.get(key, 0.0))
68
+ except Exception:
69
+ return 0.0
70
+
71
+ def supported_modes(run_config):
72
+ data = run_config.get("data", {})
73
+ modes = []
74
+ if ratio(data, "text_ratio") > 0 or "root_dir" in data:
75
+ modes.append("text")
76
+ if ratio(data, "audio_ratio") > 0 or ratio(data, "audio_only_ratio") > 0:
77
+ modes.append("audio")
78
+ if ratio(data, "audio_command_ratio") > 0 or ratio(data, "audio_command_only_ratio") > 0:
79
+ modes.append("audio_command")
80
+ if ratio(data, "video_clip_text_ratio") > 0 or ratio(data, "video_clip_only_ratio") > 0:
81
+ modes.append("video_clip")
82
+ if ratio(data, "video_ratio") > 0 or ratio(data, "video_only_ratio") > 0:
83
+ modes.append("video")
84
+ return modes
85
+
86
+ def run_sort_key(path):
87
+ match = re.search(r"(\d{10})$", path.name)
88
+ if match:
89
+ return int(match.group(1))
90
+ return int(path.stat().st_mtime)
91
+
92
+ candidates = []
93
+ for path in runs_root.glob("posttrain*.gpus8.*"):
94
+ config_path = path / "run_config.json"
95
+ if not config_path.exists():
96
+ continue
97
+ try:
98
+ run_config = json.loads(config_path.read_text())
99
+ except Exception:
100
+ continue
101
+ modes = supported_modes(run_config)
102
+ if mode == "all":
103
+ score = len([m for m in ("text", "audio", "audio_command", "video_clip", "video") if m in modes])
104
+ else:
105
+ score = 1 if mode in modes else 0
106
+ if score > 0:
107
+ candidates.append((score, run_sort_key(path), path))
108
+
109
+ if candidates:
110
+ candidates.sort()
111
+ print(str(candidates[-1][2]))
112
+ PY
113
+ )
114
+ if [ -n "$AUTO_RUN_DIR" ]; then
115
+ RUN_DIR=$AUTO_RUN_DIR
116
+ else
117
+ RUN_DIR=$DEFAULT_RUN_DIR
118
+ fi
119
+ fi
120
+
121
+ if [ -z "$RUN_DIR" ] || [ ! -f "$RUN_DIR/run_config.json" ]; then
122
+ echo "ERROR: cannot resolve a valid RUN_DIR for MODE=$MODE"
123
+ echo " Pass it explicitly: bash infer_render_auto.sh /path/to/run_dir"
124
+ exit 1
125
+ fi
126
+
127
+ SUPPORTED_MODES_CSV=$("$INFER_PY" - "$RUN_DIR" <<'PY'
128
+ import json
129
+ import sys
130
+ from pathlib import Path
131
+
132
+ run_dir = Path(sys.argv[1])
133
+ config = json.loads((run_dir / "run_config.json").read_text())
134
+ data = config.get("data", {})
135
+
136
+ def ratio(key):
137
+ try:
138
+ return float(data.get(key, 0.0))
139
+ except Exception:
140
+ return 0.0
141
+
142
+ supported = []
143
+ if ratio("text_ratio") > 0 or "root_dir" in data:
144
+ supported.append("text")
145
+ if ratio("audio_ratio") > 0 or ratio("audio_only_ratio") > 0:
146
+ supported.append("audio")
147
+ if ratio("audio_command_ratio") > 0 or ratio("audio_command_only_ratio") > 0:
148
+ supported.append("audio_command")
149
+ if ratio("video_clip_text_ratio") > 0 or ratio("video_clip_only_ratio") > 0:
150
+ supported.append("video_clip")
151
+ if ratio("video_ratio") > 0 or ratio("video_only_ratio") > 0:
152
+ supported.append("video")
153
+ print(",".join(supported))
154
+ PY
155
+ )
156
+
157
+ if [ -z "$SUPPORTED_MODES_CSV" ]; then
158
+ echo "ERROR: RUN_DIR does not expose any supported inference modes: $RUN_DIR"
159
+ exit 1
160
+ fi
161
+
162
+ IFS=',' read -r -a SUPPORTED_MODES <<< "$SUPPORTED_MODES_CSV"
163
+
164
+ mode_supported() {
165
+ local wanted
166
+ local supported
167
+ wanted=$(canonical_mode "$1")
168
+ for supported in "${SUPPORTED_MODES[@]}"; do
169
+ if [ "$supported" = "$wanted" ]; then
170
+ return 0
171
+ fi
172
+ done
173
+ return 1
174
+ }
175
+
176
+ TARGET_MODES=()
177
+ if [ "$MODE" = "all" ]; then
178
+ if mode_supported text; then
179
+ TARGET_MODES+=(text)
180
+ fi
181
+ if mode_supported audio; then
182
+ TARGET_MODES+=(audio)
183
+ fi
184
+ if mode_supported audio_command; then
185
+ TARGET_MODES+=(audio_command)
186
+ fi
187
+ if mode_supported video_clip; then
188
+ TARGET_MODES+=(video_clip)
189
+ elif mode_supported video; then
190
+ TARGET_MODES+=(video)
191
+ fi
192
+ if [ ${#TARGET_MODES[@]} -eq 0 ]; then
193
+ echo "ERROR: MODE=all but RUN_DIR has no supported target modes: $RUN_DIR"
194
+ exit 1
195
+ fi
196
+ else
197
+ if ! mode_supported "$RAW_MODE"; then
198
+ echo "ERROR: RUN_DIR does not look compatible with MODE=$RAW_MODE: $RUN_DIR"
199
+ echo " supported modes: ${SUPPORTED_MODES[*]}"
200
+ exit 1
201
+ fi
202
+ TARGET_MODES=("$RAW_MODE")
203
+ fi
204
+
205
+ CHECK_INTERVAL=10
206
+ NUM_SAMPLES=1
207
+ NUM_INFERENCE_STEPS=10
208
+ NUM_OUTPUT_FRAMES=500
209
+ AUDIO_NUM_OUTPUT_FRAMES=300
210
+ VIDEO_NUM_OUTPUT_FRAMES=300
211
+ VIDEO_CLIP_MOTION_FPS=50
212
+ DEVICE=cuda:0
213
+ GMR_PY=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/envs/gmr/bin/python
214
+ SONIC_RENDER_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/GR00T-WholeBodyControl/gear_sonic
215
+ SOURCE_FPS=50
216
+ TARGET_FPS=30
217
+
218
+ TEXT_PROMPTS=(
219
+ "A person jogs forward at a steady pace."
220
+ "A person walks forward calmly."
221
+ "A person performs a high jump and lands steadily."
222
+ "A person turns around in place."
223
+ )
224
+ TEXT_PROMPT_NAMES=(
225
+ jog_forward
226
+ walk_forward
227
+ high_jump
228
+ turn_in_place
229
+ )
230
+ TEXT_INIT_MOTION_DIRS=(
231
+ /cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/Psi0/dataset/seed_all/g1/csv_sonic/230710/jog_ff_loop_360_R_001__A419_M
232
+ /cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/Psi0/dataset/seed_all/g1/csv_sonic/230710/walk_ff_loop_315_R_001__A421_M
233
+ /cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/Psi0/dataset/seed_all/g1/csv_sonic/230710/high_jump_R_001__A418_M
234
+ /cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/Psi0/dataset/seed_all/g1/csv_sonic/230710/idle_turn_360_R_001__A418_M
235
+ )
236
+
237
+ VIDEO_MOTION_DIRS=(
238
+ /cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/GR00T-WholeBodyControl/gear_sonic_deploy/reference/converted/psi0_dance/gvhmr/movies__F-SNBR4jhY_0007
239
+ )
240
+ VIDEO_PATHS=(
241
+ /cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/dance_1723_video_psi0/dance_1723_video_10s/movies__F-SNBR4jhY_0007.mp4
242
+ )
243
+ VIDEO_PROMPTS=(
244
+ "Generate a motion sequence that matches the provided video."
245
+ )
246
+
247
+ AUDIO_MOTION_DIRS=(
248
+ /cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/GR00T-WholeBodyControl/gear_sonic_deploy/reference/converted/psi0_dance/gvhmr/movies__F-SNBR4jhY_0007
249
+ )
250
+ AUDIO_PATHS=(
251
+ /cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/dance_1723_video_psi0/dance_1723_video_10s/movies__F-SNBR4jhY_0007.mp4
252
+ )
253
+ AUDIO_PROMPTS=(
254
+ "Generate a dance sequence that matches the provided music."
255
+ )
256
+
257
+ AUDIO_COMMAND_MOTION_DIRS=(
258
+ /cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/Psi0/dataset/seed_all/g1/csv_sonic/230710/walk_ff_loop_315_R_001__A421_M
259
+ )
260
+ AUDIO_COMMAND_PATHS=(
261
+ /cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/qwen3-tts/seed_audio/003771/female_01_zh.wav
262
+ )
263
+ AUDIO_COMMAND_PROMPTS=(
264
+ "Generate a motion that follows the spoken instruction in the audio."
265
+ )
266
+
267
+ RUN_PARENT_DIR=$(dirname "$RUN_DIR")
268
+ RUN_NAME=$(basename "$RUN_DIR")
269
+ RUN_TAG=$(echo "$RUN_NAME" | awk -F. '{print $NF}')
270
+
271
+ if [ ${#TARGET_MODES[@]} -gt 1 ]; then
272
+ OUTPUT_ROOT=${PSI_AUTO_OUTPUT_DIR:-"$RUN_PARENT_DIR/$RUN_TAG.auto_infer_all"}
273
+ else
274
+ OUTPUT_ROOT=${PSI_AUTO_OUTPUT_DIR:-"$RUN_PARENT_DIR/$RUN_TAG.auto_infer.${TARGET_MODES[0]}"}
275
+ fi
276
+
277
+ output_dir_for_mode() {
278
+ local mode=$1
279
+ if [ ${#TARGET_MODES[@]} -gt 1 ]; then
280
+ echo "$OUTPUT_ROOT/$(canonical_mode "$mode")"
281
+ else
282
+ echo "$OUTPUT_ROOT"
283
+ fi
284
+ }
285
+
286
+ expected_samples_for_mode() {
287
+ case "$1" in
288
+ text|pure_text)
289
+ echo "${#TEXT_PROMPTS[@]}"
290
+ ;;
291
+ audio|audio_text|audio_only|audio_pure)
292
+ echo "${#AUDIO_MOTION_DIRS[@]}"
293
+ ;;
294
+ audio_command|audio_command_text|audio_command_only|command_audio)
295
+ echo "${#AUDIO_COMMAND_MOTION_DIRS[@]}"
296
+ ;;
297
+ video_clip|video_clip_text|video_clip_only|video|video_text|video_only)
298
+ echo "${#VIDEO_MOTION_DIRS[@]}"
299
+ ;;
300
+ *)
301
+ echo 0
302
+ ;;
303
+ esac
304
+ }
305
+
306
+ expected_motion_frames_for_mode() {
307
+ case "$1" in
308
+ text|pure_text)
309
+ echo "$NUM_OUTPUT_FRAMES"
310
+ ;;
311
+ audio|audio_text|audio_only|audio_pure|audio_command|audio_command_text|audio_command_only|command_audio)
312
+ echo "$AUDIO_NUM_OUTPUT_FRAMES"
313
+ ;;
314
+ video_clip|video_clip_text|video_clip_only|video|video_text|video_only)
315
+ echo "$VIDEO_NUM_OUTPUT_FRAMES"
316
+ ;;
317
+ *)
318
+ echo "$NUM_OUTPUT_FRAMES"
319
+ ;;
320
+ esac
321
+ }
322
+
323
+ min_render_frames_for_mode() {
324
+ local mode=$1
325
+ local motion_frames
326
+ motion_frames=$(expected_motion_frames_for_mode "$mode")
327
+ python -c 'import sys
328
+ motion_frames = int(sys.argv[1])
329
+ source_fps = float(sys.argv[2])
330
+ target_fps = float(sys.argv[3])
331
+ print(int(round((max(motion_frames, 1) - 1) / source_fps * target_fps)) + 1)' "$motion_frames" "$SOURCE_FPS" "$TARGET_FPS"
332
+ }
333
+
334
+ count_valid_render_mp4() {
335
+ local step_dir=$1
336
+ local min_render_frames=$2
337
+ local valid_render_mp4_count=0
338
+ local mp4_path
339
+ local mp4_frames
340
+
341
+ while IFS= read -r mp4_path; do
342
+ mp4_frames=$(ffprobe -v error -select_streams v:0 -show_entries stream=nb_frames -of default=noprint_wrappers=1:nokey=1 "$mp4_path" 2>/dev/null | head -n1)
343
+ if [ -n "$mp4_frames" ] && [ "$mp4_frames" -ge "$min_render_frames" ] 2>/dev/null; then
344
+ valid_render_mp4_count=$((valid_render_mp4_count + 1))
345
+ fi
346
+ done < <(find "$step_dir" -name 'reference_motion.mp4' -print 2>/dev/null)
347
+
348
+ echo "$valid_render_mp4_count"
349
+ }
350
+
351
+ run_infer_for_mode() {
352
+ local mode=$1
353
+ local ckpt_name=$2
354
+ local ckpt_step=$3
355
+ local step_dir=$4
356
+ local mode_key
357
+ local sample_i
358
+ local PROMPT
359
+ local PROMPT_NAME
360
+ local INIT_MOTION_DIR
361
+ local MOTION_DIR
362
+ local AUDIO_PATH
363
+ local VIDEO_PATH
364
+ local AUDIO_CONFIG_KEY
365
+ local AUDIO_MODE_NAME
366
+ local infer_failed=0
367
+
368
+ mode_key=$(canonical_mode "$mode")
369
+
370
+ case "$mode" in
371
+ text|pure_text)
372
+ for sample_i in "${!TEXT_PROMPTS[@]}"; do
373
+ PROMPT=${TEXT_PROMPTS[$sample_i]}
374
+ PROMPT_NAME=${TEXT_PROMPT_NAMES[$sample_i]}
375
+ INIT_MOTION_DIR=${TEXT_INIT_MOTION_DIRS[$sample_i]}
376
+ echo " [推理] 开始 $mode infer sample=$sample_i"
377
+ echo " prompt: $PROMPT"
378
+ echo " init : $INIT_MOTION_DIR"
379
+ "$INFER_PY" scripts/infer/seed_text_to_motion.py \
380
+ --run-dir "$RUN_DIR" \
381
+ --ckpt-step "$ckpt_step" \
382
+ --output-dir "$step_dir" \
383
+ --prompt "$PROMPT" \
384
+ --prompt-motion-name "$PROMPT_NAME" \
385
+ --init-motion-dir "$INIT_MOTION_DIR" \
386
+ --num-inference-steps "$NUM_INFERENCE_STEPS" \
387
+ --num-output-frames "$NUM_OUTPUT_FRAMES" \
388
+ --force-text-only \
389
+ --device "$DEVICE"
390
+ if [ $? -ne 0 ]; then
391
+ echo " [错误] $ckpt_name $mode infer失败,sample=$sample_i"
392
+ infer_failed=1
393
+ break
394
+ fi
395
+ done
396
+ ;;
397
+ audio|audio_text|audio_only|audio_pure|audio_command|audio_command_text|audio_command_only|command_audio)
398
+ if [ "$mode_key" = "audio" ]; then
399
+ AUDIO_CONFIG_KEY=audio
400
+ AUDIO_MODE_NAME=audio
401
+ MOTION_DIRS=("${AUDIO_MOTION_DIRS[@]}")
402
+ AUDIO_FILES=("${AUDIO_PATHS[@]}")
403
+ PROMPTS=("${AUDIO_PROMPTS[@]}")
404
+ else
405
+ AUDIO_CONFIG_KEY=audio_command
406
+ AUDIO_MODE_NAME=audio_command
407
+ MOTION_DIRS=("${AUDIO_COMMAND_MOTION_DIRS[@]}")
408
+ AUDIO_FILES=("${AUDIO_COMMAND_PATHS[@]}")
409
+ PROMPTS=("${AUDIO_COMMAND_PROMPTS[@]}")
410
+ fi
411
+
412
+ for sample_i in "${!MOTION_DIRS[@]}"; do
413
+ MOTION_DIR=${MOTION_DIRS[$sample_i]}
414
+ AUDIO_PATH=${AUDIO_FILES[$sample_i]}
415
+ PROMPT=${PROMPTS[$sample_i]}
416
+
417
+ echo " [推理] 开始 $mode infer sample=$sample_i"
418
+ echo " audio_config: $AUDIO_CONFIG_KEY"
419
+ echo " motion : $MOTION_DIR"
420
+ echo " audio : $AUDIO_PATH"
421
+ "$INFER_PY" scripts/infer/seed_audio_to_motion.py \
422
+ --run-dir "$RUN_DIR" \
423
+ --ckpt-step "$ckpt_step" \
424
+ --output-dir "$step_dir" \
425
+ --motion-dir "$MOTION_DIR" \
426
+ --video-path "$AUDIO_PATH" \
427
+ --audio-config-key "$AUDIO_CONFIG_KEY" \
428
+ --prompt "$PROMPT" \
429
+ --num-inference-steps "$NUM_INFERENCE_STEPS" \
430
+ --num-output-frames "$AUDIO_NUM_OUTPUT_FRAMES" \
431
+ --device "$DEVICE"
432
+ if [ $? -ne 0 ]; then
433
+ echo " [错误] $ckpt_name $AUDIO_MODE_NAME infer失败,sample=$sample_i"
434
+ infer_failed=1
435
+ break
436
+ fi
437
+ done
438
+ ;;
439
+ video_clip|video_clip_text|video_clip_only|video|video_text|video_only)
440
+ for sample_i in "${!VIDEO_MOTION_DIRS[@]}"; do
441
+ MOTION_DIR=${VIDEO_MOTION_DIRS[$sample_i]}
442
+ VIDEO_PATH=${VIDEO_PATHS[$sample_i]}
443
+ PROMPT=${VIDEO_PROMPTS[$sample_i]}
444
+ VIDEO_ARGS=(--video-config-key video)
445
+
446
+ if [ "$mode_key" = "video_clip" ]; then
447
+ VIDEO_ARGS=(--video-config-key video_clip --video-clip --video-clip-motion-fps "$VIDEO_CLIP_MOTION_FPS")
448
+ fi
449
+ if [ "$mode" = "video_only" ] || [ "$mode" = "video_clip_only" ]; then
450
+ PROMPT=""
451
+ fi
452
+
453
+ echo " [推理] 开始 $mode infer sample=$sample_i"
454
+ echo " motion: $MOTION_DIR"
455
+ echo " video : $VIDEO_PATH"
456
+ "$INFER_PY" scripts/infer/seed_video_to_motion.py \
457
+ --run-dir "$RUN_DIR" \
458
+ --ckpt-step "$ckpt_step" \
459
+ --output-dir "$step_dir" \
460
+ --motion-dir "$MOTION_DIR" \
461
+ --video-path "$VIDEO_PATH" \
462
+ --prompt "$PROMPT" \
463
+ --num-inference-steps "$NUM_INFERENCE_STEPS" \
464
+ --num-output-frames "$VIDEO_NUM_OUTPUT_FRAMES" \
465
+ "${VIDEO_ARGS[@]}" \
466
+ --device "$DEVICE"
467
+ if [ $? -ne 0 ]; then
468
+ echo " [错误] $ckpt_name $mode infer失败,sample=$sample_i"
469
+ infer_failed=1
470
+ break
471
+ fi
472
+ done
473
+ ;;
474
+ *)
475
+ echo " [错误] unknown mode: $mode"
476
+ infer_failed=1
477
+ ;;
478
+ esac
479
+
480
+ if [ "$infer_failed" -ne 0 ]; then
481
+ return 1
482
+ fi
483
+ return 0
484
+ }
485
+
486
+ run_render_for_mode() {
487
+ local mode=$1
488
+ local ckpt_name=$2
489
+ local step_dir=$3
490
+ local expected_samples=$4
491
+ local min_render_frames=$5
492
+ local valid_render_mp4_count
493
+
494
+ echo " [渲染] 开始 $mode render"
495
+ (
496
+ cd "$SONIC_RENDER_ROOT"
497
+ if [ -n "$XVFB_RUN" ]; then
498
+ env -u PYTHONPATH \
499
+ PYOPENGL_PLATFORM=egl \
500
+ MUJOCO_GL=egl \
501
+ QT_X11_NO_MITSHM=1 \
502
+ SONIC_RENDER_HEADLESS_CAMERA=1 \
503
+ SONIC_RENDER_LOCAL_TMP_OUTPUT=1 \
504
+ SONIC_RENDER_TMPDIR=/tmp \
505
+ "$XVFB_RUN" -a -s "-screen 0 1280x1024x24 -extension MIT-SHM" \
506
+ "$GMR_PY" scripts/batch_render_reference_motion_mujoco.py \
507
+ --motion-dir "$step_dir" \
508
+ --source-fps "$SOURCE_FPS" \
509
+ --target-fps "$TARGET_FPS" \
510
+ --dof-order isaaclab
511
+ else
512
+ echo " [渲染] xvfb-run not found; use headless EGL camera"
513
+ env -u PYTHONPATH \
514
+ PYOPENGL_PLATFORM=egl \
515
+ MUJOCO_GL=egl \
516
+ QT_X11_NO_MITSHM=1 \
517
+ SONIC_RENDER_HEADLESS_CAMERA=1 \
518
+ SONIC_RENDER_LOCAL_TMP_OUTPUT=1 \
519
+ SONIC_RENDER_TMPDIR=/tmp \
520
+ "$GMR_PY" scripts/batch_render_reference_motion_mujoco.py \
521
+ --motion-dir "$step_dir" \
522
+ --source-fps "$SOURCE_FPS" \
523
+ --target-fps "$TARGET_FPS" \
524
+ --dof-order isaaclab
525
+ fi
526
+ )
527
+
528
+ if [ $? -ne 0 ]; then
529
+ valid_render_mp4_count=$(count_valid_render_mp4 "$step_dir" "$min_render_frames")
530
+ if [ "$valid_render_mp4_count" -ge "$expected_samples" ]; then
531
+ echo " [警告] $mode render返回非0,但已生成 ${valid_render_mp4_count}/${expected_samples} 个有效reference_motion.mp4,视为完成"
532
+ else
533
+ echo " [错误] $ckpt_name $mode render失败,当前有效reference_motion.mp4数量: $valid_render_mp4_count/${expected_samples}"
534
+ return 1
535
+ fi
536
+ else
537
+ echo " [完成] $ckpt_name $mode render成功"
538
+ fi
539
+ return 0
540
+ }
541
+
542
+ process_mode_for_ckpt() {
543
+ local mode=$1
544
+ local ckpt_name=$2
545
+ local ckpt_step=$3
546
+ local output_dir
547
+ local step_dir
548
+ local expected_samples
549
+ local infer_csv_count
550
+ local min_render_frames
551
+ local valid_render_mp4_count
552
+
553
+ output_dir=$(output_dir_for_mode "$mode")
554
+ step_dir="$output_dir/step_$ckpt_step"
555
+ expected_samples=$(expected_samples_for_mode "$mode")
556
+ min_render_frames=$(min_render_frames_for_mode "$mode")
557
+
558
+ mkdir -p "$step_dir"
559
+
560
+ infer_csv_count=$(find "$step_dir" -name 'body_pos.csv' -print 2>/dev/null | wc -l | tr -d ' ')
561
+ valid_render_mp4_count=$(count_valid_render_mp4 "$step_dir" "$min_render_frames")
562
+
563
+ echo " [模式] $mode"
564
+ echo " 输出目录: $step_dir"
565
+ echo " infer: ${infer_csv_count}/${expected_samples}"
566
+ echo " render: ${valid_render_mp4_count}/${expected_samples}"
567
+
568
+ if [ "$infer_csv_count" -lt "$expected_samples" ]; then
569
+ if ! run_infer_for_mode "$mode" "$ckpt_name" "$ckpt_step" "$step_dir"; then
570
+ return 1
571
+ fi
572
+ infer_csv_count=$(find "$step_dir" -name 'body_pos.csv' -print 2>/dev/null | wc -l | tr -d ' ')
573
+ echo " [完成] $ckpt_name $mode infer成功,当前body_pos.csv数量: $infer_csv_count"
574
+ else
575
+ echo " [跳过] $mode infer已完成: ${infer_csv_count}/${expected_samples} 个body_pos.csv"
576
+ fi
577
+
578
+ valid_render_mp4_count=$(count_valid_render_mp4 "$step_dir" "$min_render_frames")
579
+ if [ "$valid_render_mp4_count" -lt "$expected_samples" ]; then
580
+ if ! run_render_for_mode "$mode" "$ckpt_name" "$step_dir" "$expected_samples" "$min_render_frames"; then
581
+ return 1
582
+ fi
583
+ else
584
+ echo " [跳过] $mode render已完成: ${valid_render_mp4_count}/${expected_samples} 个有效reference_motion.mp4"
585
+ fi
586
+
587
+ return 0
588
+ }
589
+
590
+ if [ -x /usr/bin/xvfb-run ]; then
591
+ XVFB_RUN=/usr/bin/xvfb-run
592
+ elif [ -x /bin/xvfb-run ]; then
593
+ XVFB_RUN=/bin/xvfb-run
594
+ elif command -v xvfb-run >/dev/null 2>&1; then
595
+ XVFB_RUN=$(command -v xvfb-run)
596
+ else
597
+ XVFB_RUN=
598
+ fi
599
+
600
+ export PYOPENGL_PLATFORM=egl
601
+ export MUJOCO_GL=egl
602
+ export QT_X11_NO_MITSHM=1
603
+
604
+ echo "=========================================="
605
+ echo "Auto Infer + Render Monitor"
606
+ echo "=========================================="
607
+ echo "RUN_DIR: $RUN_DIR"
608
+ echo "OUTPUT_ROOT: $OUTPUT_ROOT"
609
+ echo "RAW_MODE: $RAW_MODE"
610
+ echo "MODE: $MODE"
611
+ echo "SUPPORTED_MODES: ${SUPPORTED_MODES[*]}"
612
+ echo "TARGET_MODES: ${TARGET_MODES[*]}"
613
+ echo "CHECK_INTERVAL: ${CHECK_INTERVAL}s"
614
+ echo "NUM_OUTPUT_FRAMES: $NUM_OUTPUT_FRAMES"
615
+ echo "AUDIO_NUM_OUTPUT_FRAMES: $AUDIO_NUM_OUTPUT_FRAMES"
616
+ echo "VIDEO_NUM_OUTPUT_FRAMES: $VIDEO_NUM_OUTPUT_FRAMES"
617
+ echo "NUM_INFERENCE_STEPS: $NUM_INFERENCE_STEPS"
618
+ echo "XVFB_RUN: ${XVFB_RUN:-<not found, use headless EGL camera>}"
619
+ echo "NOTE: default MODE=all. Set PSI_AUTO_INFER_MODE to run a single modality."
620
+ echo "NOTE: single-mode values: text / audio / audio_command / video_clip"
621
+ echo "=========================================="
622
+
623
+ mkdir -p "$OUTPUT_ROOT"
624
+ for mode in "${TARGET_MODES[@]}"; do
625
+ mode_output_dir=$(output_dir_for_mode "$mode")
626
+ mkdir -p "$mode_output_dir"
627
+ echo "OUTPUT_DIR[$mode]: $mode_output_dir"
628
+ done
629
+
630
+ while true; do
631
+ echo ""
632
+ echo "[$(date '+%Y-%m-%d %H:%M:%S')] 扫描新的checkpoint..."
633
+
634
+ pending_ckpts=()
635
+
636
+ checkpoint_dirs=()
637
+ for ckpt_dir in "$RUN_DIR"/checkpoints/ckpt_*; do
638
+ [ -d "$ckpt_dir" ] || continue
639
+ checkpoint_dirs+=("$ckpt_dir")
640
+ done
641
+ if [ ${#checkpoint_dirs[@]} -eq 0 ] && [ -d "$RUN_DIR/checkpoints/best_ckpt" ]; then
642
+ checkpoint_dirs+=("$RUN_DIR/checkpoints/best_ckpt")
643
+ fi
644
+
645
+ for ckpt_dir in "${checkpoint_dirs[@]}"; do
646
+ ckpt_name=$(basename "$ckpt_dir")
647
+ if [ "$ckpt_name" = "best_ckpt" ]; then
648
+ ckpt_step=best
649
+ sort_step=999999999999
650
+ elif [[ "$ckpt_name" =~ ckpt_([0-9]+) ]]; then
651
+ ckpt_step="${BASH_REMATCH[1]}"
652
+ sort_step="$ckpt_step"
653
+ else
654
+ echo " [警告] 无法从 $ckpt_name 提取step数,跳过"
655
+ continue
656
+ fi
657
+
658
+ ckpt_pending_modes=()
659
+ for scan_mode in "${TARGET_MODES[@]}"; do
660
+ scan_step_dir="$(output_dir_for_mode "$scan_mode")/step_$ckpt_step"
661
+ scan_expected_samples=$(expected_samples_for_mode "$scan_mode")
662
+ scan_min_render_frames=$(min_render_frames_for_mode "$scan_mode")
663
+ scan_valid_render_mp4_count=$(count_valid_render_mp4 "$scan_step_dir" "$scan_min_render_frames")
664
+ if [ "$scan_valid_render_mp4_count" -lt "$scan_expected_samples" ]; then
665
+ ckpt_pending_modes+=("$scan_mode")
666
+ fi
667
+ done
668
+
669
+ if [ ${#ckpt_pending_modes[@]} -eq 0 ]; then
670
+ echo " [跳过] $ckpt_name (所有目标模态都已完成)"
671
+ continue
672
+ fi
673
+
674
+ pending_modes_csv=$(IFS=,; echo "${ckpt_pending_modes[*]}")
675
+ pending_ckpts+=("${sort_step}|${ckpt_step}|${ckpt_dir}|${pending_modes_csv}")
676
+ done
677
+
678
+ if [ ${#pending_ckpts[@]} -eq 0 ]; then
679
+ echo " [无] 没有待处理的checkpoint"
680
+ echo "[$(date '+%Y-%m-%d %H:%M:%S')] 等待 ${CHECK_INTERVAL} 秒后再次检查..."
681
+ sleep "$CHECK_INTERVAL"
682
+ continue
683
+ fi
684
+
685
+ best_item=$(printf '%s\n' "${pending_ckpts[@]}" | sort -t'|' -k1,1nr | head -n1)
686
+ IFS='|' read -r sort_step ckpt_step ckpt_dir pending_modes_csv <<< "$best_item"
687
+ ckpt_name=$(basename "$ckpt_dir")
688
+ IFS=',' read -r -a CKPT_PENDING_MODES <<< "$pending_modes_csv"
689
+
690
+ echo " [发现] 待处理checkpoint共 ${#pending_ckpts[@]} 个,选择step最大的:"
691
+ echo " $ckpt_name"
692
+ echo " [步骤] step: $ckpt_step"
693
+ echo " [模式] 待处理: ${CKPT_PENDING_MODES[*]}"
694
+
695
+ mode_failed=0
696
+ for mode in "${CKPT_PENDING_MODES[@]}"; do
697
+ if ! process_mode_for_ckpt "$mode" "$ckpt_name" "$ckpt_step"; then
698
+ mode_failed=1
699
+ break
700
+ fi
701
+ done
702
+
703
+ if [ "$mode_failed" -ne 0 ]; then
704
+ echo "[$(date '+%Y-%m-%d %H:%M:%S')] 等待 ${CHECK_INTERVAL} 秒后再次检查..."
705
+ sleep "$CHECK_INTERVAL"
706
+ continue
707
+ fi
708
+
709
+ echo "[$(date '+%Y-%m-%d %H:%M:%S')] 等待 ${CHECK_INTERVAL} 秒后再次检查..."
710
+ sleep "$CHECK_INTERVAL"
711
+ done
plan_0.md ADDED
@@ -0,0 +1,374 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Psi0 改造成 Text-to-Humanoid-Motion 计划(最小改动版)
2
+
3
+ ## 0) 目标与约束
4
+
5
+ 你的目标可以抽象成:
6
+
7
+ 1. 输入:文本 caption(先不使用图像/音频)。
8
+ 2. 输出:机器人动作序列(你定义为每帧 36 维,`29 dof + xyz + wxyz`)。
9
+ 3. 训练策略:不走 pretrain,直接用 posttrain 框架,冻结 VLM,只训练 action expert(扩散动作头)。
10
+ 4. 未来扩展:VLM 换到 `Qwen-2.5-Omni-3B`,当前先做 text-only smoke test。
11
+ 5. 工程要求:尽可能少改现有代码。
12
+
13
+ 下面的方案按这个目标严格展开。
14
+
15
+ ---
16
+
17
+ ## 1) 建议先看的代码(阅读顺序)
18
+
19
+ 按这个顺序看,最快建立全局心智模型:
20
+
21
+ 1. 训练主入口与配置注入
22
+ - `scripts/train.py`(训练循环、配置解析入口)
23
+ - `src/psi/config/config.py`(`LaunchConfig/TrainConfig/DataConfig/ModelConfig`)
24
+ - `src/psi/config/train/posttrain_he_psi0_config.py`
25
+ - `src/psi/config/train/posttrain_mix_psi0_config.py`
26
+ - `scripts/train/psi0/posttrain-he-psi0.sh`
27
+
28
+ 2. posttrain trainer 主流程
29
+ - `src/psi/trainers/posttrain.py`
30
+ - `src/psi/trainers/trainer.py`
31
+
32
+ 3. 数据层(raw -> transform -> batch)
33
+ - `src/psi/config/data_he.py`
34
+ - `src/psi/data/humanoid/he_raw_dataset.py`
35
+ - `src/psi/data/dataset.py`
36
+ - `src/psi/config/transform.py`(重点:`HEPosttrainRepackTransform`, `ActionStateTransform`, `Psi0ModelTransform`)
37
+ - `src/psi/trainers/qwen3vl_mixin.py`(`PaddedCollatorForTogether`)
38
+
39
+ 4. 模型层(VLM + action expert)
40
+ - `src/psi/config/model_psi0.py`
41
+ - `src/psi/models/psi0.py`(重点:`Psi0Model.forward`, `ActionTransformerModel`, `ObservationProjection`)
42
+
43
+ 5. 数据预处理脚本(参考)
44
+ - `scripts/data/raw_he_to_psi0.py`
45
+ - `scripts/data/raw_to_lerobot.py`
46
+ - `scripts/data/calc_modality_stats.py`
47
+ - `scripts/data/raw_dexmate_psi0.py`
48
+
49
+ ---
50
+
51
+ ## 2) 当前 posttrain 数据流和训练流(从数据处理开始)
52
+
53
+ ## 2.1 训练入口与实例化
54
+
55
+ 1. `scripts/train.py` 根据命令行第一个参数动态导入 config 模块(例如 `posttrain_he_psi0_config`)。
56
+ 2. `Trainer.instantiate` 会按 `cfg.train.name` 选择 `psi.trainers.posttrain.PosttrainTrainer`。
57
+ 3. 流程是固定的:`init_models -> create_datasets -> create_dataloaders -> create_optimizer_and_scheduler -> prepare -> training loop`。
58
+
59
+ 对应代码:
60
+ - `scripts/train.py:169-219, 233-314, 350-360`
61
+
62
+ ## 2.2 数据集原始样本(HE 例子)
63
+
64
+ `HERawDataset.__getitem__` 输出 raw dict,核心字段:
65
+
66
+ 1. `observation.images.egocentric`: `(To, 3, H, W)`,默认 `To=1`。
67
+ 2. `observation.hand_joints`, `observation.arm_joints`。
68
+ 3. 展平后的 action 字段(例如 `action.joint_angles`)。
69
+ 4. `task`(instruction 文本),`action_mask`,`obs_mask`。
70
+
71
+ 对应代码:
72
+ - `src/psi/data/humanoid/he_raw_dataset.py:176-237`
73
+
74
+ ## 2.3 三段 transform:repack -> field -> model
75
+
76
+ `MapStyleDataset.__getitem__` 会执行:
77
+
78
+ 1. `repack`:把 raw 字段整理成统一结构。
79
+ 2. `field`:做动作/状态归一化、padding、反归一化逻辑。
80
+ 3. `model`:把图像+文本喂给 processor,生成 `input_ids/pixel_values/...`。
81
+
82
+ 对应代码:
83
+ - `src/psi/data/dataset.py:23-26`
84
+ - `src/psi/config/transform.py`(`DataTransform.__call__`)
85
+
86
+ ### repack(posttrain HE)
87
+
88
+ `HEPosttrainRepackTransform` 做的事:
89
+
90
+ 1. 构造 `states`、`actions`。
91
+ 2. 如果 `use_delta_actions=True`,将动作转成 delta(`actions[1:] - actions[:-1]`)。
92
+ 3. 按 `pad_action_dim/pad_state_dim` 补齐。
93
+ 4. 输出统一字段:`observations`, `states`, `actions`, `instruction`, `actions_mask` 等。
94
+
95
+ 对应代码:
96
+ - `src/psi/config/transform.py:705-767`
97
+
98
+ ### field(动作归一化)
99
+
100
+ `ActionStateTransform`:
101
+
102
+ 1. 从 `stat_path` 读 `min/max` 或 `q01/q99`。
103
+ 2. 对 `actions` 归一化到 `[-1,1]`。
104
+ 3. 保留 `raw_actions`,给评估和反归一化使用。
105
+
106
+ 对应代码:
107
+ - `src/psi/config/transform.py:408-578`
108
+
109
+ ### model(构造 VLM 输入)
110
+
111
+ `Psi0ModelTransform`:
112
+
113
+ 1. 图像 resize/crop/aug。
114
+ 2. 构造 chat message:`[images] + text instruction`。
115
+ 3. `vlm_processor(...)` 得到 `input_ids`, `attention_mask`, `pixel_values`, `image_grid_thw`。
116
+ 4. 附带 `actions/states/actions_mask/raw_actions` 回传。
117
+
118
+ 对应代码:
119
+ - `src/psi/config/transform.py:797-888`
120
+
121
+ ## 2.4 collator 到 batch
122
+
123
+ `PaddedCollatorForTogether` 负责:
124
+
125
+ 1. 对 `input_ids` padding。
126
+ 2. 组装 `attention_mask`。
127
+ 3. 强制 stack `pixel_values` 和 `image_grid_thw`。
128
+ 4. stack `actions/states/actions_mask`。
129
+
130
+ 这意味着目前代码路径默认“有视觉输入字段”。
131
+
132
+ 对应代码:
133
+ - `src/psi/trainers/qwen3vl_mixin.py:87-157`
134
+
135
+ ## 2.5 模型 forward(posttrain)
136
+
137
+ `PosttrainTrainer.forward_and_loss` 里调用:
138
+
139
+ 1. 从 GT 动作加噪(flow 或 ddpm)。
140
+ 2. `Psi0Model.forward(...)` 输入:
141
+ - `input_ids`, `attention_mask`, `pixel_values`, `image_grid_thw`
142
+ - `action_samples`(噪声动作)
143
+ - `states`
144
+ - `timestep`
145
+ 3. `Psi0Model` ��跑 VLM 拿最后一层 hidden states,再喂 action expert(扩散动作头)。
146
+ 4. 输出 `action_pred`,与目标噪声/速度算 MSE。
147
+
148
+ 对应代码:
149
+ - `src/psi/trainers/posttrain.py:525-584`
150
+ - `src/psi/models/psi0.py:1602-1639`
151
+
152
+ ## 2.6 训练损失与评估
153
+
154
+ 1. loss 是动作维度 MSE,加 `actions_mask` 后按维度加权汇总。
155
+ 2. `action_dim != 7` 时,默认均匀权重(你的 36 维会走这个分支)。
156
+ 3. 评估里目前有固定分组切分 `[7,14,21]`,这是历史结构假设,对你的 36 维语义不一定成立。
157
+
158
+ 对应代码:
159
+ - `src/psi/trainers/posttrain.py:55-64`
160
+ - `src/psi/trainers/posttrain.py:505-517`
161
+
162
+ ---
163
+
164
+ ## 3) 输入输出维度分别在哪里定义
165
+
166
+ ### 3.1 动作/状态主维度
167
+
168
+ 1. `--model.action-dim`:动作维度(你要设 36)。
169
+ 2. `--model.odim`:状态维度(要和最终 `states` 最后一维一致)。
170
+ 3. `--model.action-chunk-size`:每次预测 horizon(`Tp`)。
171
+ 4. `--data.transform.repack.pad-action-dim`:repack 后动作 pad 到多少维。
172
+ 5. `--data.transform.repack.pad-state-dim`:repack 后状态 pad 到多少维。
173
+ 6. `--data.transform.field.pad-action-dim`:归一化统计也按这个维度对齐。
174
+ 7. `--data.transform.field.pad-state-dim`:同上。
175
+
176
+ 参考:
177
+ - `scripts/train/psi0/posttrain-he-psi0.sh:55-75`
178
+ - `src/psi/config/model_psi0.py:17-37`
179
+ - `src/psi/models/psi0.py:592-594, 757-761, 788, 1499-1503`
180
+
181
+ ### 3.2 VLM hidden size 与 view_feature_dim
182
+
183
+ 1. `ObservationProjection.views_proj = Linear(view_feature_dim -> hidden_dim)`。
184
+ 2. 这里的 `view_feature_dim` 必须等于 VLM 最后一层 hidden size。
185
+ 3. 当前脚本用 `--model.view_feature_dim=2048`(对应现有 Qwen3-VL-2B 配置)。
186
+
187
+ 参考:
188
+ - `src/psi/models/psi0.py:499`
189
+ - `src/psi/models/psi0.py:1624, 1632`
190
+ - `scripts/train/psi0/posttrain-he-psi0.sh:75`
191
+
192
+ ### 3.3 典型 batch 形状(当前 posttrain he 脚本)
193
+
194
+ 1. `actions`: `(B, 16, 36)`
195
+ 2. `states`: `(B, 1, 36)`(`observation_horizon=1`)
196
+ 3. `input_ids`: `(B, L)`
197
+ 4. `attention_mask`: `(B, L)`
198
+ 5. `pixel_values`, `image_grid_thw`: 由 processor 决定
199
+
200
+ ---
201
+
202
+ ## 4) 面向你目标的最小改造方案
203
+
204
+ 下面给两条路径,我建议先 A 再 B。
205
+
206
+ ## 路径 A(最小改动,优先跑通)
207
+
208
+ 核心思想:先保留现有视觉字段接口,不做“真正 text-only 张量路径”;数据侧喂一个固定占位图(dummy image),让模型主要学 text->motion。这样能最大复用 `Psi0ModelTransform + Collator + Psi0Model.forward`,代码改动最小。
209
+
210
+ 你需要做:
211
+
212
+ 1. 新增 `BoneSeedDataset`(或在现有 dataset 适配)输出统一字段:
213
+ - `observations`: 放一个固定 PIL 图(例如纯黑 240x320)
214
+ - `states`: `(To, 36)`,通常 `To=1`
215
+ - `actions`: `(Tp, 36)`
216
+ - `instruction`: caption
217
+ - `actions_mask`: `(Tp, 36)`
218
+
219
+ 2. 新增一个 `DataConfig`(例如 `data_boneseed.py`)和一个训练 config(例如 `posttrain_boneseed_psi0_config.py`):
220
+ - `transform.repack`: 你的 `BoneSeedRepackTransform`
221
+ - `transform.field`: 复用 `ActionStateTransform`
222
+ - `transform.model`: 先复用 `Psi0ModelTransform`
223
+
224
+ 3. 新增训练脚本(仿 `posttrain-he-psi0.sh`):
225
+ - 维度全设 36
226
+ - 先设置 `n_conditions=0`
227
+ - `tune_vlm=False`
228
+ - `noise_scheduler=flow`
229
+
230
+ 优点:
231
+
232
+ 1. 几乎不动模型与 trainer 主干。
233
+ 2. 能最快 smoke test。
234
+
235
+ 代价:
236
+
237
+ 1. 严格来说不是“纯 text-only”实现,只是视觉分支恒定输入。
238
+
239
+ ## 路径 B(真正 text-only)
240
+
241
+ 需要改 3 个点:
242
+
243
+ 1. `Psi0ModelTransform.build_qwenvl_inputs`:
244
+ - 允许 `imgs=[]`。
245
+ - `process_vision_info` 返回 `None` 时,仍可仅返回 text tokens。
246
+
247
+ 2. `PaddedCollatorForTogether`:
248
+ - `pixel_values/image_grid_thw` 改成可选字段(存在才 stack)。
249
+
250
+ 3. `Psi0Model.forward`:
251
+ - `pixel_values=None, image_grid_thw=None` 时也能调用 VLM。
252
+ - 对应 forward 参数改成 optional。
253
+
254
+ 这是更“干净”的 text-to-motion 路径,但改动比 A 稍大。
255
+
256
+ ---
257
+
258
+ ## 5) Qwen-2.5-Omni-3B 替换要点
259
+
260
+ 当前 posttrain 写死了 Qwen3 类:
261
+
262
+ 1. `src/psi/trainers/posttrain.py` 里 import `Qwen3VLForConditionalGeneration`。
263
+ 2. `src/psi/models/psi0.py` 类型标注也是 `Qwen3VLForConditionalGeneration`。
264
+
265
+ 要换 Omni,最小做法:
266
+
267
+ 1. 在 posttrain 里改成可切换 VLM 构造:
268
+ - `Qwen2_5OmniForConditionalGeneration` + `Qwen2_5OmniProcessor`(或 Auto 类统一封装)。
269
+
270
+ 2. 保持 `output_hidden_states=True`,继续取最后一层 hidden states 作为 action expert 条件。
271
+
272
+ 3. 校对 hidden size,并同步 `--model.view_feature_dim`。
273
+
274
+ 你当前环境可用性(已验证):
275
+
276
+ 1. `.venv-psi` 下 `transformers==4.57.0`。
277
+ 2. 可 import:
278
+ - `Qwen2_5OmniForConditionalGeneration`
279
+ - `Qwen2_5OmniProcessor`
280
+
281
+ 建议你自己再做一次快速确认:
282
+
283
+ ```bash
284
+ .venv-psi/bin/python - <<'PY'
285
+ from transformers import Qwen2_5OmniForConditionalGeneration, Qwen2_5OmniProcessor
286
+ print("ok")
287
+ PY
288
+ ```
289
+
290
+ 再确认 hidden size(用于 `view_feature_dim`):
291
+
292
+ ```bash
293
+ .venv-psi/bin/python - <<'PY'
294
+ from transformers import AutoConfig
295
+ cfg = AutoConfig.from_pretrained("Qwen/Qwen2.5-Omni-3B")
296
+ print("thinker hidden:", getattr(getattr(cfg, "thinker_config", None), "hidden_size", None))
297
+ print("hidden:", getattr(cfg, "hidden_size", None))
298
+ PY
299
+ ```
300
+
301
+ ---
302
+
303
+ ## 6) 从数据处理开始的落地步骤(建议执行顺序)
304
+
305
+ 1. 先定动作语义:
306
+ - 你的 36 维是绝对量还是 delta。
307
+ - 如果你数据已经是重定向后的绝对关节+位姿,优先 `use_delta_actions=False`,先简化问题。
308
+
309
+ 2. 做 dataset 适配(最关键):
310
+ - 先写 `BoneSeedDataset.__getitem__`,把每个样本变成统一结构。
311
+ - 确保 `states/actions` 维度稳定。
312
+ - caption 映射到 `instruction`。
313
+
314
+ 3. 生成统计文件(归一化必需):
315
+ - `ActionStateTransform` 需要 `stat_path`。
316
+ - 至少包含 `action` 键下的 `min/max` 或 `q01/q99`,维度 36。
317
+
318
+ 4. 新建专用 config 与训练脚本:
319
+ - 先复用 posttrain trainer,不要改训练循环。
320
+ - 参数先直接对齐:`action_dim=36`, `odim=36`, `pad_*_dim=36`。
321
+
322
+ 5. 再换 VLM 到 Omni:
323
+ - 先只做“文本+占位图”跑通。
324
+ - 然后再做严格 text-only 可选字段改造。
325
+
326
+ 6. 最后做 smoke test:
327
+ - 1-2 个 batch overfit(`overfit_single_batch=True`)先看 loss 能不能快速下降。
328
+
329
+ ---
330
+
331
+ ## 7) 你现在就要注意的风险点
332
+
333
+ 1. `HEPosttrainRepackTransform` 里有明显 typo:
334
+ - `obs_mask": datpa["obs_mask"]`,`datpa` 未定义。
335
+ - 位置:`src/psi/config/transform.py:766`
336
+ - 如果你复用这段逻辑,先修掉。
337
+
338
+ 2. 评估指标分组写死(对 36 维不语义一致):
339
+ - `np.split(..., [7,14,21])`
340
+ - 位置:`src/psi/trainers/posttrain.py:512-517`
341
+ - 训练不一定受影响,但日志会误导。
342
+
343
+ 3. 当前 collator/forward 对视觉字段是强依赖:
344
+ - 不改的话,text-only 会在 batch 组装处报错。
345
+ - 位置:`src/psi/trainers/qwen3vl_mixin.py:115-125`, `src/psi/models/psi0.py:1604-1622`
346
+
347
+ 4. `view_feature_dim` 和 VLM hidden size 必须一致:
348
+ - 不一致会在线性层处 shape mismatch。
349
+ - 位置:`src/psi/models/psi0.py:499`
350
+
351
+ ---
352
+
353
+ ## 8) 最小改造文件清单(建议)
354
+
355
+ 建议新增(而不是改老逻辑):
356
+
357
+ 1. `src/psi/data/humanoid/boneseed_dataset.py`
358
+ 2. `src/psi/config/data_boneseed.py`
359
+ 3. `src/psi/config/train/posttrain_boneseed_psi0_config.py`
360
+ 4. `scripts/train/psi0/posttrain-boneseed-psi0.sh`
361
+
362
+ 如要做严格 text-only,再改:
363
+
364
+ 1. `src/psi/config/transform.py`(`Psi0ModelTransform`)
365
+ 2. `src/psi/trainers/qwen3vl_mixin.py`(`PaddedCollatorForTogether`)
366
+ 3. `src/psi/models/psi0.py`(`Psi0Model.forward` optional vision args)
367
+ 4. `src/psi/trainers/posttrain.py`(VLM 类替换成 Omni)
368
+
369
+ ---
370
+
371
+ ## 9) 一句话执行策略
372
+
373
+ 先用“占位图 + 冻结 VLM + 只训 action expert”把 36 维 caption->motion 流程跑通,再把视觉输入改成 truly optional(text-only),最后替换到 Qwen2.5-Omni 并校准 `view_feature_dim`。
374
+
posttrain-v1-psi0-all-7task-tune-vlm-direct-dense-actionhead-L.sh ADDED
@@ -0,0 +1,313 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ pkill -f monitor.py
2
+
3
+ cd /cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/Psi0-yth-v21
4
+ export PYTHONPATH=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/Psi0-yth-v21/src
5
+ export CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7
6
+ export PSI_OUTPUT_DIR=${PSI_OUTPUT_DIR:-/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/Psi0-yth-v21/runs}
7
+ export PSI_LOG_REPORT_TO=wandb
8
+
9
+ source /cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/Psi0-yth-v15/.venv-psi/bin/activate
10
+ [ -f .env ] && source .env
11
+
12
+ export WANDB_API_KEY="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"
13
+
14
+ export OMP_NUM_THREADS=8
15
+ export TF_CPP_MIN_LOG_LEVEL=3
16
+
17
+ # Keep JSON sources readable in this script, but pass compact JSON to tyro.
18
+ compact_json() {
19
+ python -c 'import json, sys; print(json.dumps(json.load(sys.stdin), separators=(",", ":")))'
20
+ }
21
+
22
+ PSI_SEED_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/Psi0/dataset/seed_all
23
+ PSI_SEED_METADATA=metadata/seed_metadata_v004.csv
24
+
25
+ PSI_DANCE_MOTION_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/GR00T-WholeBodyControl/gear_sonic_deploy/reference/converted/psi0_dance/gvhmr
26
+ PSI_DANCE_AUDIO_VIDEO_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/dance_1723_video_psi0/dance_1723_video_10s
27
+ PSI_DANCE_VIDEO_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/dance_1723_video_psi0/dance_1723_video_10s
28
+ PSI_DANCE_VIDEO_EMBED_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/dance_1723_video_psi0/dance_1723_video_10s_embeds
29
+ PSI_VIDEO_DEMO_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/videogen/motion_example_0506_demo
30
+ PSI_QWEN3_TTS_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/qwen3-tts
31
+ PSI_VIDEO_CLIP_DATASET_ROOT=${PSI_VIDEO_CLIP_DATASET_ROOT:-/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/videogen/psi0_video_clip_50f_dense_stride1_direct_full}
32
+ PSI_DANCE_VIDEO_CLIP_ROOT=$PSI_VIDEO_CLIP_DATASET_ROOT/dance_1723_video_10s
33
+ PSI_MOTION_EXAMPLE_VIDEO_CLIP_ROOT=$PSI_VIDEO_CLIP_DATASET_ROOT/motion_example_0506_demo
34
+ PSI_DANCE_VIDEO_CLIP_EMBED_ROOT=${PSI_DANCE_VIDEO_CLIP_EMBED_ROOT:-$PSI_DANCE_VIDEO_CLIP_ROOT/embeds}
35
+ PSI_MOTION_EXAMPLE_VIDEO_CLIP_EMBED_ROOT=${PSI_MOTION_EXAMPLE_VIDEO_CLIP_EMBED_ROOT:-$PSI_MOTION_EXAMPLE_VIDEO_CLIP_ROOT/embeds}
36
+ PSI_MOTION_EXAMPLE_VIDEO_EMBED_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/videogen/motion_example_0506_demo_video_embeds
37
+
38
+ if [ -z "$PSI_AUDIO_SOURCES_JSON" ]; then
39
+ PSI_AUDIO_SOURCES_JSON=$(compact_json <<JSON
40
+ [
41
+ {
42
+ "name": "psi0_dance_audio",
43
+ "root_dir": "$PSI_DANCE_MOTION_ROOT",
44
+ "video_dir": "$PSI_DANCE_AUDIO_VIDEO_ROOT",
45
+ "dataset_name": "seed-audio-g1"
46
+ }
47
+ ]
48
+ JSON
49
+ )
50
+ fi
51
+
52
+ if [ -z "$PSI_AUDIO_COMMAND_SOURCES_JSON" ]; then
53
+ PSI_AUDIO_COMMAND_SOURCES_JSON=$(compact_json <<JSON
54
+ [
55
+ {
56
+ "name": "qwen3_tts_seed_audio_command",
57
+ "root_dir": "$PSI_SEED_ROOT",
58
+ "dataset_root": "$PSI_QWEN3_TTS_ROOT",
59
+ "metadata_jsonl": "metadata.jsonl",
60
+ "motion_dir_key": "filename",
61
+ "audio_path_key": "audio_path",
62
+ "instruction_key": "content_short_description",
63
+ "motion_metadata_csv": "$PSI_SEED_METADATA",
64
+ "motion_metadata_key": "filename",
65
+ "motion_metadata_path_key": "move_g1_path",
66
+ "audio_file_extensions": ".wav",
67
+ "audio_task": "command",
68
+ "dataset_name": "seed-audio-command-g1"
69
+ }
70
+ ]
71
+ JSON
72
+ )
73
+ fi
74
+
75
+ if [ -z "$PSI_VIDEO_SOURCES_JSON" ]; then
76
+ PSI_VIDEO_SOURCES_JSON=$(compact_json <<JSON
77
+ [
78
+ {
79
+ "name": "psi0_dance_gvhmr",
80
+ "root_dir": "$PSI_DANCE_MOTION_ROOT",
81
+ "video_dir": "$PSI_DANCE_VIDEO_ROOT",
82
+ "video_embed_dir": "$PSI_DANCE_VIDEO_EMBED_ROOT",
83
+ "dataset_name": "seed-video-g1"
84
+ },
85
+ {
86
+ "name": "motion_example_0506_demo",
87
+ "dataset_root": "$PSI_VIDEO_DEMO_ROOT",
88
+ "metadata_jsonl": "metadata.jsonl",
89
+ "motion_dir_key": "converted_path",
90
+ "video_path_key": "video_path",
91
+ "instruction_key": "video_caption",
92
+ "video_embed_dir": "$PSI_MOTION_EXAMPLE_VIDEO_EMBED_ROOT",
93
+ "dataset_name": "motion-example-video-g1"
94
+ }
95
+ ]
96
+ JSON
97
+ )
98
+ fi
99
+
100
+ if [ -z "$PSI_VIDEO_CLIP_SOURCES_JSON" ]; then
101
+ PSI_VIDEO_CLIP_SOURCES_JSON=$(compact_json <<JSON
102
+ [
103
+ {
104
+ "name": "dance_1723_video_10s_clip50",
105
+ "dataset_root": "$PSI_DANCE_VIDEO_CLIP_ROOT",
106
+ "metadata_jsonl": "metadata.jsonl",
107
+ "motion_dir_key": "converted_path",
108
+ "video_path_key": "video_path",
109
+ "instruction_key": "video_caption",
110
+ "video_embed_dir": "$PSI_DANCE_VIDEO_CLIP_EMBED_ROOT",
111
+ "dataset_name": "dance-video-clip-g1"
112
+ },
113
+ {
114
+ "name": "motion_example_0506_demo_clip50",
115
+ "dataset_root": "$PSI_MOTION_EXAMPLE_VIDEO_CLIP_ROOT",
116
+ "metadata_jsonl": "metadata.jsonl",
117
+ "motion_dir_key": "converted_path",
118
+ "video_path_key": "video_path",
119
+ "instruction_key": "video_caption",
120
+ "video_embed_dir": "$PSI_MOTION_EXAMPLE_VIDEO_CLIP_EMBED_ROOT",
121
+ "dataset_name": "motion-example-video-clip-g1"
122
+ }
123
+ ]
124
+ JSON
125
+ )
126
+ fi
127
+
128
+ # 7-task from-scratch mix, excluding legacy VIDEO_TEXT / VIDEO_ONLY.
129
+ # Smaller data sources get larger weights. These use a tempered inverse-size
130
+ # rule, (47M / source_size)^0.25, so rare modalities are upweighted without
131
+ # completely starving text/video_clip in BatchMixtureSampler.
132
+ # text stride1 frame samples: ~47,000,000
133
+ # music audio dance frame samples: 213,731
134
+ # audio_command wav files: 41,760
135
+ # dense video_clip metadata: 4,597,576
136
+ PSI_TEXT_TASK_WEIGHT=1.0
137
+ PSI_AUDIO_TEXT_TASK_WEIGHT=3.8509
138
+ PSI_AUDIO_ONLY_TASK_WEIGHT=3.8509
139
+ PSI_AUDIO_COMMAND_TEXT_TASK_WEIGHT=5.7921
140
+ PSI_AUDIO_COMMAND_ONLY_TASK_WEIGHT=5.7921
141
+ PSI_VIDEO_TEXT_TASK_WEIGHT=0
142
+ PSI_VIDEO_ONLY_TASK_WEIGHT=0
143
+ PSI_VIDEO_CLIP_TEXT_TASK_WEIGHT=1.7881
144
+ PSI_VIDEO_CLIP_ONLY_TASK_WEIGHT=1.7881
145
+
146
+ PSI_PRETRAIN_BACKBONE_PATH=/cpfs01/VideoGen/public_ckpt/Qwen/Qwen2.5-Omni-3B
147
+ NUM_PAST_FRAMES=0
148
+ STATE_HORIZON=$((NUM_PAST_FRAMES + 1))
149
+ ACTION_CHUNK_SIZE=30
150
+ ACTION_EXEC_HORIZON=20
151
+ PROGRESS_DIM=5
152
+ ACTION_HIDDEN_DIM=1920
153
+ ACTION_NUM_BLOCKS=28
154
+ ACTION_NHEADS=20
155
+ ACTION_HEAD_DIM=$((ACTION_HIDDEN_DIM / ACTION_NHEADS))
156
+ VIDEO_SAMPLE_FPS=1.0
157
+ VIDEO_CLIP_SAMPLE_FPS=8.0
158
+ VIDEO_SIZE_PIXELS=50176
159
+ VIDEO_DECODER=ffmpeg
160
+ VIDEO_CACHE_SIZE=64
161
+ VIDEO_CLIP_MOTION_CACHE_SIZE=256
162
+ VIDEO_CLIP_TRUST_METADATA=${VIDEO_CLIP_TRUST_METADATA:-1}
163
+ VIDEO_CLIP_RECORDS_CACHE_DIR=${VIDEO_CLIP_RECORDS_CACHE_DIR:-$PSI_VIDEO_CLIP_DATASET_ROOT/.index_cache}
164
+ if [ "$VIDEO_CLIP_TRUST_METADATA" = "1" ] || [ "$VIDEO_CLIP_TRUST_METADATA" = "true" ] || [ "$VIDEO_CLIP_TRUST_METADATA" = "True" ]; then
165
+ VIDEO_CLIP_TRUST_METADATA_FLAG="--data.video-clip.trust-metadata"
166
+ else
167
+ VIDEO_CLIP_TRUST_METADATA_FLAG="--data.video-clip.no-trust-metadata"
168
+ fi
169
+ DATALOADER_NUM_WORKERS=4
170
+ VAL_DATALOADER_NUM_WORKERS=2
171
+
172
+ NPROC_PER_NODE=$(echo $CUDA_VISIBLE_DEVICES | tr ',' '\n' | wc -l)
173
+
174
+ echo "Training with $NPROC_PER_NODE GPU(s): $CUDA_VISIBLE_DEVICES"
175
+ echo "Text root: $PSI_SEED_ROOT"
176
+ echo "Dance motion root: $PSI_DANCE_MOTION_ROOT"
177
+ echo "Dance audio/video root: $PSI_DANCE_AUDIO_VIDEO_ROOT"
178
+ echo "Dance video root: $PSI_DANCE_VIDEO_ROOT"
179
+ echo "Dance video embed root: $PSI_DANCE_VIDEO_EMBED_ROOT"
180
+ echo "Qwen3-TTS acdudio command root: $PSI_QWEN3_TTS_ROOT"
181
+ echo "Audio sources: $PSI_AUDIO_SOURCES_JSON"
182
+ echo "Audio command sources: $PSI_AUDIO_COMMAND_SOURCES_JSON"
183
+ echo "Video sources: $PSI_VIDEO_SOURCES_JSON"
184
+ echo "Video clip sources: $PSI_VIDEO_CLIP_SOURCES_JSON"
185
+ echo "Task weights: text=$PSI_TEXT_TASK_WEIGHT audio_text=$PSI_AUDIO_TEXT_TASK_WEIGHT audio_only=$PSI_AUDIO_ONLY_TASK_WEIGHT audio_command_text=$PSI_AUDIO_COMMAND_TEXT_TASK_WEIGHT audio_command_only=$PSI_AUDIO_COMMAND_ONLY_TASK_WEIGHT video_text=$PSI_VIDEO_TEXT_TASK_WEIGHT video_only=$PSI_VIDEO_ONLY_TASK_WEIGHT video_clip_text=$PSI_VIDEO_CLIP_TEXT_TASK_WEIGHT video_clip_only=$PSI_VIDEO_CLIP_ONLY_TASK_WEIGHT"
186
+ # echo "Resume run dir: $PSI_PRETRAIN_RUN_DIR"
187
+ echo "TextOp 39D action/state enabled"
188
+ echo "Future progress embedding: enabled, dim $PROGRESS_DIM"
189
+ echo "Action head scale: L hidden=$ACTION_HIDDEN_DIM depth=$ACTION_NUM_BLOCKS heads=$ACTION_NHEADS head_dim=$ACTION_HEAD_DIM"
190
+ echo "Backbone path: $PSI_PRETRAIN_BACKBONE_PATH"
191
+ echo "Output dir: $PSI_OUTPUT_DIR"
192
+ echo "Video clip trust metadata: $VIDEO_CLIP_TRUST_METADATA"
193
+ echo "Video clip records cache dir: $VIDEO_CLIP_RECORDS_CACHE_DIR"
194
+
195
+ args="posttrain_v1_seed_text_audio_video_mix_psi0_config \
196
+ --seed=292285 \
197
+ --exp=posttrain_actionhead_L \
198
+ --timestamp=$(date +"%y%m%d%H%M") \
199
+ --train.name=posttrain_v1_trimodal \
200
+ --train.data_parallel=ddp \
201
+ --train.mixed_precision=bf16 \
202
+ --train.train_batch_size=16 \
203
+ --train.val_batch_size=32 \
204
+ --train.num-workers=$DATALOADER_NUM_WORKERS \
205
+ --train.val-num-workers=$VAL_DATALOADER_NUM_WORKERS \
206
+ --train.resume_from_checkpoint=None \
207
+ --train.max_checkpoints_to_keep=1 \
208
+ --train.gradient_accumulation_steps=1 \
209
+ --train.learning_rate=1e-4 \
210
+ --train.max_training_steps=500000 \
211
+ --train.warmup_ratio=None \
212
+ --train.warmup_steps=1000 \
213
+ --train.checkpointing_steps=1000 \
214
+ --train.validation_steps=1000 \
215
+ --train.val_num_batches=20 \
216
+ --train.max_grad_norm=1.0 \
217
+ --train.lr_scheduler_type=cosine_with_min_lr \
218
+ --train.lr_scheduler_kwargs.betas 0.9 0.999 \
219
+ --train.lr_scheduler_kwargs.weight_decay=0 \
220
+ --train.lr_scheduler_kwargs.eps=1e-8 \
221
+ --log.report_to=$PSI_LOG_REPORT_TO \
222
+ --train.output_dir=$PSI_OUTPUT_DIR \
223
+ --data.text.root-dir=$PSI_SEED_ROOT \
224
+ --data.text.metadata-csv=$PSI_SEED_METADATA \
225
+ --data.text-ratio=$PSI_TEXT_TASK_WEIGHT \
226
+ --data.audio.sources-json=$PSI_AUDIO_SOURCES_JSON \
227
+ --data.audio-ratio=$PSI_AUDIO_TEXT_TASK_WEIGHT \
228
+ --data.audio-only-ratio=$PSI_AUDIO_ONLY_TASK_WEIGHT \
229
+ --data.audio.sample-stride=1 \
230
+ --data.audio.motion-fps=50 \
231
+ --data.audio.audio-sampling-rate=16000 \
232
+ --data.audio.audio-window-seconds=1.0 \
233
+ --data.audio.audio-left-context-seconds=0.0 \
234
+ --data.audio.full-audio \
235
+ --data.audio-command.sources-json=$PSI_AUDIO_COMMAND_SOURCES_JSON \
236
+ --data.audio-command-ratio=$PSI_AUDIO_COMMAND_TEXT_TASK_WEIGHT \
237
+ --data.audio-command-only-ratio=$PSI_AUDIO_COMMAND_ONLY_TASK_WEIGHT \
238
+ --data.audio-command.sample-stride=1 \
239
+ --data.audio-command.motion-fps=50 \
240
+ --data.audio-command.audio-sampling-rate=16000 \
241
+ --data.audio-command.audio-window-seconds=1.0 \
242
+ --data.audio-command.audio-left-context-seconds=0.0 \
243
+ --data.audio-command.full-audio \
244
+ --data.video-ratio=$PSI_VIDEO_TEXT_TASK_WEIGHT \
245
+ --data.video-only-ratio=$PSI_VIDEO_ONLY_TASK_WEIGHT \
246
+ --data.video.sample-stride=1 \
247
+ --data.video.motion-fps=50 \
248
+ --data.video.video-sample-fps=$VIDEO_SAMPLE_FPS \
249
+ --data.video.video-size-pixels=$VIDEO_SIZE_PIXELS \
250
+ --data.video.video-decoder=$VIDEO_DECODER \
251
+ --data.video.video-cache-size=$VIDEO_CACHE_SIZE \
252
+ --data.video.sources-json=$PSI_VIDEO_SOURCES_JSON \
253
+ --data.video-clip-text-ratio=$PSI_VIDEO_CLIP_TEXT_TASK_WEIGHT \
254
+ --data.video-clip-only-ratio=$PSI_VIDEO_CLIP_ONLY_TASK_WEIGHT \
255
+ --data.video-clip.sources-json=$PSI_VIDEO_CLIP_SOURCES_JSON \
256
+ --data.video-clip.sample-stride=1 \
257
+ --data.video-clip.motion-fps=50 \
258
+ --data.video-clip.video-sample-fps=$VIDEO_CLIP_SAMPLE_FPS \
259
+ --data.video-clip.video-size-pixels=$VIDEO_SIZE_PIXELS \
260
+ --data.video-clip.video-decoder=$VIDEO_DECODER \
261
+ --data.video-clip.video-cache-size=$VIDEO_CACHE_SIZE \
262
+ --data.video-clip.motion-cache-size=$VIDEO_CLIP_MOTION_CACHE_SIZE \
263
+ $VIDEO_CLIP_TRUST_METADATA_FLAG \
264
+ --data.video-clip.records-cache-dir=$VIDEO_CLIP_RECORDS_CACHE_DIR \
265
+ --data.video-clip.action-exec-horizon=$ACTION_EXEC_HORIZON \
266
+ --data.sampler=batch_mixture \
267
+ --data.val-task=text,audio,audio_only,audio_command,audio_command_only,video_clip,video_clip_only \
268
+ --data.transform.repack.num-past-frames=$NUM_PAST_FRAMES \
269
+ --data.transform.repack.textop-actions \
270
+ --data.transform.repack.action-chunk-size=$ACTION_CHUNK_SIZE \
271
+ --data.transform.repack.pad-action-dim=39 \
272
+ --data.transform.repack.pad-state-dim=39 \
273
+ --data.transform.field.action_norm_type=bounds_q99 \
274
+ --data.transform.field.stat-path=assets/stats/seed_all_g1_motion39_textop_stats.json \
275
+ --data.transform.field.normalize-state \
276
+ --data.transform.field.pad-action-dim=39 \
277
+ --data.transform.field.pad-state-dim=39 \
278
+ --data.transform.model.resize.size 240 320 \
279
+ --data.transform.model.center_crop.size 240 320 \
280
+ --data.transform.model.no-img-aug \
281
+ --data.transform.model.text-only \
282
+ --data.transform.model.audio-only \
283
+ --data.transform.model.video-only \
284
+ --data.transform.model.video-size-pixels=$VIDEO_SIZE_PIXELS \
285
+ --model.model_name_or_path=$PSI_PRETRAIN_BACKBONE_PATH \
286
+ --model.noise-scheduler=flow \
287
+ --model.n_conditions=0 \
288
+ --model.action-chunk-size=$ACTION_CHUNK_SIZE \
289
+ --model.action-dim=39 \
290
+ --model.action-exec-horizon=$ACTION_EXEC_HORIZON \
291
+ --model.observation-horizon=$STATE_HORIZON \
292
+ --model.odim=39 \
293
+ --model.view_feature_dim=2048 \
294
+ --model.hidden-dim=$ACTION_HIDDEN_DIM \
295
+ --model.num-blocks=$ACTION_NUM_BLOCKS \
296
+ --model.nhead=$ACTION_NHEADS \
297
+ --model.use-future-progress-embedding \
298
+ --model.future-progress-dim=$PROGRESS_DIM \
299
+ --model.tune-vlm \
300
+ --model.lang-backbone-lr=1e-5 \
301
+ --model.no-use_film \
302
+ --model.no-combined_temb \
303
+ --model.rtc \
304
+ --model.max-delay=11 \
305
+ --train.max-grad-value=1 \
306
+ --train.loss-group-weight.joint-abs=1.0 \
307
+ --train.loss-group-weight.root-delta-xy=3.0 \
308
+ --train.loss-group-weight.root-rot5-delta=10.0 \
309
+ --train.loss-group-weight.root-height=15.0 \
310
+ --train.loss-group-weight.foot-contact=0.1
311
+ "
312
+ torchrun --nproc_per_node=$NPROC_PER_NODE --master_port=29500 scripts/train.py \
313
+ $args "$@"
posttrain-v1-psi0-all-7task-tune-vlm-direct-dense-actionhead-M.sh ADDED
@@ -0,0 +1,313 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ pkill -f monitor.py
2
+
3
+ cd /cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/Psi0-yth-v21
4
+ export PYTHONPATH=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/Psi0-yth-v21/src
5
+ export CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7
6
+ export PSI_OUTPUT_DIR=${PSI_OUTPUT_DIR:-/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/Psi0-yth-v21/runs}
7
+ export PSI_LOG_REPORT_TO=wandb
8
+
9
+ source /cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/Psi0-yth-v15/.venv-psi/bin/activate
10
+ [ -f .env ] && source .env
11
+
12
+ export WANDB_API_KEY="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"
13
+
14
+ export OMP_NUM_THREADS=8
15
+ export TF_CPP_MIN_LOG_LEVEL=3
16
+
17
+ # Keep JSON sources readable in this script, but pass compact JSON to tyro.
18
+ compact_json() {
19
+ python -c 'import json, sys; print(json.dumps(json.load(sys.stdin), separators=(",", ":")))'
20
+ }
21
+
22
+ PSI_SEED_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/Psi0/dataset/seed_all
23
+ PSI_SEED_METADATA=metadata/seed_metadata_v004.csv
24
+
25
+ PSI_DANCE_MOTION_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/GR00T-WholeBodyControl/gear_sonic_deploy/reference/converted/psi0_dance/gvhmr
26
+ PSI_DANCE_AUDIO_VIDEO_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/dance_1723_video_psi0/dance_1723_video_10s
27
+ PSI_DANCE_VIDEO_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/dance_1723_video_psi0/dance_1723_video_10s
28
+ PSI_DANCE_VIDEO_EMBED_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/dance_1723_video_psi0/dance_1723_video_10s_embeds
29
+ PSI_VIDEO_DEMO_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/videogen/motion_example_0506_demo
30
+ PSI_QWEN3_TTS_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/qwen3-tts
31
+ PSI_VIDEO_CLIP_DATASET_ROOT=${PSI_VIDEO_CLIP_DATASET_ROOT:-/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/videogen/psi0_video_clip_50f_dense_stride1_direct_full}
32
+ PSI_DANCE_VIDEO_CLIP_ROOT=$PSI_VIDEO_CLIP_DATASET_ROOT/dance_1723_video_10s
33
+ PSI_MOTION_EXAMPLE_VIDEO_CLIP_ROOT=$PSI_VIDEO_CLIP_DATASET_ROOT/motion_example_0506_demo
34
+ PSI_DANCE_VIDEO_CLIP_EMBED_ROOT=${PSI_DANCE_VIDEO_CLIP_EMBED_ROOT:-$PSI_DANCE_VIDEO_CLIP_ROOT/embeds}
35
+ PSI_MOTION_EXAMPLE_VIDEO_CLIP_EMBED_ROOT=${PSI_MOTION_EXAMPLE_VIDEO_CLIP_EMBED_ROOT:-$PSI_MOTION_EXAMPLE_VIDEO_CLIP_ROOT/embeds}
36
+ PSI_MOTION_EXAMPLE_VIDEO_EMBED_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/videogen/motion_example_0506_demo_video_embeds
37
+
38
+ if [ -z "$PSI_AUDIO_SOURCES_JSON" ]; then
39
+ PSI_AUDIO_SOURCES_JSON=$(compact_json <<JSON
40
+ [
41
+ {
42
+ "name": "psi0_dance_audio",
43
+ "root_dir": "$PSI_DANCE_MOTION_ROOT",
44
+ "video_dir": "$PSI_DANCE_AUDIO_VIDEO_ROOT",
45
+ "dataset_name": "seed-audio-g1"
46
+ }
47
+ ]
48
+ JSON
49
+ )
50
+ fi
51
+
52
+ if [ -z "$PSI_AUDIO_COMMAND_SOURCES_JSON" ]; then
53
+ PSI_AUDIO_COMMAND_SOURCES_JSON=$(compact_json <<JSON
54
+ [
55
+ {
56
+ "name": "qwen3_tts_seed_audio_command",
57
+ "root_dir": "$PSI_SEED_ROOT",
58
+ "dataset_root": "$PSI_QWEN3_TTS_ROOT",
59
+ "metadata_jsonl": "metadata.jsonl",
60
+ "motion_dir_key": "filename",
61
+ "audio_path_key": "audio_path",
62
+ "instruction_key": "content_short_description",
63
+ "motion_metadata_csv": "$PSI_SEED_METADATA",
64
+ "motion_metadata_key": "filename",
65
+ "motion_metadata_path_key": "move_g1_path",
66
+ "audio_file_extensions": ".wav",
67
+ "audio_task": "command",
68
+ "dataset_name": "seed-audio-command-g1"
69
+ }
70
+ ]
71
+ JSON
72
+ )
73
+ fi
74
+
75
+ if [ -z "$PSI_VIDEO_SOURCES_JSON" ]; then
76
+ PSI_VIDEO_SOURCES_JSON=$(compact_json <<JSON
77
+ [
78
+ {
79
+ "name": "psi0_dance_gvhmr",
80
+ "root_dir": "$PSI_DANCE_MOTION_ROOT",
81
+ "video_dir": "$PSI_DANCE_VIDEO_ROOT",
82
+ "video_embed_dir": "$PSI_DANCE_VIDEO_EMBED_ROOT",
83
+ "dataset_name": "seed-video-g1"
84
+ },
85
+ {
86
+ "name": "motion_example_0506_demo",
87
+ "dataset_root": "$PSI_VIDEO_DEMO_ROOT",
88
+ "metadata_jsonl": "metadata.jsonl",
89
+ "motion_dir_key": "converted_path",
90
+ "video_path_key": "video_path",
91
+ "instruction_key": "video_caption",
92
+ "video_embed_dir": "$PSI_MOTION_EXAMPLE_VIDEO_EMBED_ROOT",
93
+ "dataset_name": "motion-example-video-g1"
94
+ }
95
+ ]
96
+ JSON
97
+ )
98
+ fi
99
+
100
+ if [ -z "$PSI_VIDEO_CLIP_SOURCES_JSON" ]; then
101
+ PSI_VIDEO_CLIP_SOURCES_JSON=$(compact_json <<JSON
102
+ [
103
+ {
104
+ "name": "dance_1723_video_10s_clip50",
105
+ "dataset_root": "$PSI_DANCE_VIDEO_CLIP_ROOT",
106
+ "metadata_jsonl": "metadata.jsonl",
107
+ "motion_dir_key": "converted_path",
108
+ "video_path_key": "video_path",
109
+ "instruction_key": "video_caption",
110
+ "video_embed_dir": "$PSI_DANCE_VIDEO_CLIP_EMBED_ROOT",
111
+ "dataset_name": "dance-video-clip-g1"
112
+ },
113
+ {
114
+ "name": "motion_example_0506_demo_clip50",
115
+ "dataset_root": "$PSI_MOTION_EXAMPLE_VIDEO_CLIP_ROOT",
116
+ "metadata_jsonl": "metadata.jsonl",
117
+ "motion_dir_key": "converted_path",
118
+ "video_path_key": "video_path",
119
+ "instruction_key": "video_caption",
120
+ "video_embed_dir": "$PSI_MOTION_EXAMPLE_VIDEO_CLIP_EMBED_ROOT",
121
+ "dataset_name": "motion-example-video-clip-g1"
122
+ }
123
+ ]
124
+ JSON
125
+ )
126
+ fi
127
+
128
+ # 7-task from-scratch mix, excluding legacy VIDEO_TEXT / VIDEO_ONLY.
129
+ # Smaller data sources get larger weights. These use a tempered inverse-size
130
+ # rule, (47M / source_size)^0.25, so rare modalities are upweighted without
131
+ # completely starving text/video_clip in BatchMixtureSampler.
132
+ # text stride1 frame samples: ~47,000,000
133
+ # music audio dance frame samples: 213,731
134
+ # audio_command wav files: 41,760
135
+ # dense video_clip metadata: 4,597,576
136
+ PSI_TEXT_TASK_WEIGHT=1.0
137
+ PSI_AUDIO_TEXT_TASK_WEIGHT=3.8509
138
+ PSI_AUDIO_ONLY_TASK_WEIGHT=3.8509
139
+ PSI_AUDIO_COMMAND_TEXT_TASK_WEIGHT=5.7921
140
+ PSI_AUDIO_COMMAND_ONLY_TASK_WEIGHT=5.7921
141
+ PSI_VIDEO_TEXT_TASK_WEIGHT=0
142
+ PSI_VIDEO_ONLY_TASK_WEIGHT=0
143
+ PSI_VIDEO_CLIP_TEXT_TASK_WEIGHT=1.7881
144
+ PSI_VIDEO_CLIP_ONLY_TASK_WEIGHT=1.7881
145
+
146
+ PSI_PRETRAIN_BACKBONE_PATH=/cpfs01/VideoGen/public_ckpt/Qwen/Qwen2.5-Omni-3B
147
+ NUM_PAST_FRAMES=0
148
+ STATE_HORIZON=$((NUM_PAST_FRAMES + 1))
149
+ ACTION_CHUNK_SIZE=30
150
+ ACTION_EXEC_HORIZON=20
151
+ PROGRESS_DIM=5
152
+ ACTION_HIDDEN_DIM=1152
153
+ ACTION_NUM_BLOCKS=16
154
+ ACTION_NHEADS=12
155
+ ACTION_HEAD_DIM=$((ACTION_HIDDEN_DIM / ACTION_NHEADS))
156
+ VIDEO_SAMPLE_FPS=1.0
157
+ VIDEO_CLIP_SAMPLE_FPS=8.0
158
+ VIDEO_SIZE_PIXELS=50176
159
+ VIDEO_DECODER=ffmpeg
160
+ VIDEO_CACHE_SIZE=64
161
+ VIDEO_CLIP_MOTION_CACHE_SIZE=256
162
+ VIDEO_CLIP_TRUST_METADATA=${VIDEO_CLIP_TRUST_METADATA:-1}
163
+ VIDEO_CLIP_RECORDS_CACHE_DIR=${VIDEO_CLIP_RECORDS_CACHE_DIR:-$PSI_VIDEO_CLIP_DATASET_ROOT/.index_cache}
164
+ if [ "$VIDEO_CLIP_TRUST_METADATA" = "1" ] || [ "$VIDEO_CLIP_TRUST_METADATA" = "true" ] || [ "$VIDEO_CLIP_TRUST_METADATA" = "True" ]; then
165
+ VIDEO_CLIP_TRUST_METADATA_FLAG="--data.video-clip.trust-metadata"
166
+ else
167
+ VIDEO_CLIP_TRUST_METADATA_FLAG="--data.video-clip.no-trust-metadata"
168
+ fi
169
+ DATALOADER_NUM_WORKERS=4
170
+ VAL_DATALOADER_NUM_WORKERS=2
171
+
172
+ NPROC_PER_NODE=$(echo $CUDA_VISIBLE_DEVICES | tr ',' '\n' | wc -l)
173
+
174
+ echo "Training with $NPROC_PER_NODE GPU(s): $CUDA_VISIBLE_DEVICES"
175
+ echo "Text root: $PSI_SEED_ROOT"
176
+ echo "Dance motion root: $PSI_DANCE_MOTION_ROOT"
177
+ echo "Dance audio/video root: $PSI_DANCE_AUDIO_VIDEO_ROOT"
178
+ echo "Dance video root: $PSI_DANCE_VIDEO_ROOT"
179
+ echo "Dance video embed root: $PSI_DANCE_VIDEO_EMBED_ROOT"
180
+ echo "Qwen3-TTS acdudio command root: $PSI_QWEN3_TTS_ROOT"
181
+ echo "Audio sources: $PSI_AUDIO_SOURCES_JSON"
182
+ echo "Audio command sources: $PSI_AUDIO_COMMAND_SOURCES_JSON"
183
+ echo "Video sources: $PSI_VIDEO_SOURCES_JSON"
184
+ echo "Video clip sources: $PSI_VIDEO_CLIP_SOURCES_JSON"
185
+ echo "Task weights: text=$PSI_TEXT_TASK_WEIGHT audio_text=$PSI_AUDIO_TEXT_TASK_WEIGHT audio_only=$PSI_AUDIO_ONLY_TASK_WEIGHT audio_command_text=$PSI_AUDIO_COMMAND_TEXT_TASK_WEIGHT audio_command_only=$PSI_AUDIO_COMMAND_ONLY_TASK_WEIGHT video_text=$PSI_VIDEO_TEXT_TASK_WEIGHT video_only=$PSI_VIDEO_ONLY_TASK_WEIGHT video_clip_text=$PSI_VIDEO_CLIP_TEXT_TASK_WEIGHT video_clip_only=$PSI_VIDEO_CLIP_ONLY_TASK_WEIGHT"
186
+ # echo "Resume run dir: $PSI_PRETRAIN_RUN_DIR"
187
+ echo "TextOp 39D action/state enabled"
188
+ echo "Future progress embedding: enabled, dim $PROGRESS_DIM"
189
+ echo "Action head scale: M hidden=$ACTION_HIDDEN_DIM depth=$ACTION_NUM_BLOCKS heads=$ACTION_NHEADS head_dim=$ACTION_HEAD_DIM"
190
+ echo "Backbone path: $PSI_PRETRAIN_BACKBONE_PATH"
191
+ echo "Output dir: $PSI_OUTPUT_DIR"
192
+ echo "Video clip trust metadata: $VIDEO_CLIP_TRUST_METADATA"
193
+ echo "Video clip records cache dir: $VIDEO_CLIP_RECORDS_CACHE_DIR"
194
+
195
+ args="posttrain_v1_seed_text_audio_video_mix_psi0_config \
196
+ --seed=292285 \
197
+ --exp=posttrain_actionhead_M \
198
+ --timestamp=$(date +"%y%m%d%H%M") \
199
+ --train.name=posttrain_v1_trimodal \
200
+ --train.data_parallel=ddp \
201
+ --train.mixed_precision=bf16 \
202
+ --train.train_batch_size=32 \
203
+ --train.val_batch_size=64 \
204
+ --train.num-workers=$DATALOADER_NUM_WORKERS \
205
+ --train.val-num-workers=$VAL_DATALOADER_NUM_WORKERS \
206
+ --train.resume_from_checkpoint=None \
207
+ --train.max_checkpoints_to_keep=1 \
208
+ --train.gradient_accumulation_steps=1 \
209
+ --train.learning_rate=1e-4 \
210
+ --train.max_training_steps=500000 \
211
+ --train.warmup_ratio=None \
212
+ --train.warmup_steps=1000 \
213
+ --train.checkpointing_steps=1000 \
214
+ --train.validation_steps=1000 \
215
+ --train.val_num_batches=20 \
216
+ --train.max_grad_norm=1.0 \
217
+ --train.lr_scheduler_type=cosine_with_min_lr \
218
+ --train.lr_scheduler_kwargs.betas 0.9 0.999 \
219
+ --train.lr_scheduler_kwargs.weight_decay=0 \
220
+ --train.lr_scheduler_kwargs.eps=1e-8 \
221
+ --log.report_to=$PSI_LOG_REPORT_TO \
222
+ --train.output_dir=$PSI_OUTPUT_DIR \
223
+ --data.text.root-dir=$PSI_SEED_ROOT \
224
+ --data.text.metadata-csv=$PSI_SEED_METADATA \
225
+ --data.text-ratio=$PSI_TEXT_TASK_WEIGHT \
226
+ --data.audio.sources-json=$PSI_AUDIO_SOURCES_JSON \
227
+ --data.audio-ratio=$PSI_AUDIO_TEXT_TASK_WEIGHT \
228
+ --data.audio-only-ratio=$PSI_AUDIO_ONLY_TASK_WEIGHT \
229
+ --data.audio.sample-stride=1 \
230
+ --data.audio.motion-fps=50 \
231
+ --data.audio.audio-sampling-rate=16000 \
232
+ --data.audio.audio-window-seconds=1.0 \
233
+ --data.audio.audio-left-context-seconds=0.0 \
234
+ --data.audio.full-audio \
235
+ --data.audio-command.sources-json=$PSI_AUDIO_COMMAND_SOURCES_JSON \
236
+ --data.audio-command-ratio=$PSI_AUDIO_COMMAND_TEXT_TASK_WEIGHT \
237
+ --data.audio-command-only-ratio=$PSI_AUDIO_COMMAND_ONLY_TASK_WEIGHT \
238
+ --data.audio-command.sample-stride=1 \
239
+ --data.audio-command.motion-fps=50 \
240
+ --data.audio-command.audio-sampling-rate=16000 \
241
+ --data.audio-command.audio-window-seconds=1.0 \
242
+ --data.audio-command.audio-left-context-seconds=0.0 \
243
+ --data.audio-command.full-audio \
244
+ --data.video-ratio=$PSI_VIDEO_TEXT_TASK_WEIGHT \
245
+ --data.video-only-ratio=$PSI_VIDEO_ONLY_TASK_WEIGHT \
246
+ --data.video.sample-stride=1 \
247
+ --data.video.motion-fps=50 \
248
+ --data.video.video-sample-fps=$VIDEO_SAMPLE_FPS \
249
+ --data.video.video-size-pixels=$VIDEO_SIZE_PIXELS \
250
+ --data.video.video-decoder=$VIDEO_DECODER \
251
+ --data.video.video-cache-size=$VIDEO_CACHE_SIZE \
252
+ --data.video.sources-json=$PSI_VIDEO_SOURCES_JSON \
253
+ --data.video-clip-text-ratio=$PSI_VIDEO_CLIP_TEXT_TASK_WEIGHT \
254
+ --data.video-clip-only-ratio=$PSI_VIDEO_CLIP_ONLY_TASK_WEIGHT \
255
+ --data.video-clip.sources-json=$PSI_VIDEO_CLIP_SOURCES_JSON \
256
+ --data.video-clip.sample-stride=1 \
257
+ --data.video-clip.motion-fps=50 \
258
+ --data.video-clip.video-sample-fps=$VIDEO_CLIP_SAMPLE_FPS \
259
+ --data.video-clip.video-size-pixels=$VIDEO_SIZE_PIXELS \
260
+ --data.video-clip.video-decoder=$VIDEO_DECODER \
261
+ --data.video-clip.video-cache-size=$VIDEO_CACHE_SIZE \
262
+ --data.video-clip.motion-cache-size=$VIDEO_CLIP_MOTION_CACHE_SIZE \
263
+ $VIDEO_CLIP_TRUST_METADATA_FLAG \
264
+ --data.video-clip.records-cache-dir=$VIDEO_CLIP_RECORDS_CACHE_DIR \
265
+ --data.video-clip.action-exec-horizon=$ACTION_EXEC_HORIZON \
266
+ --data.sampler=batch_mixture \
267
+ --data.val-task=text,audio,audio_only,audio_command,audio_command_only,video_clip,video_clip_only \
268
+ --data.transform.repack.num-past-frames=$NUM_PAST_FRAMES \
269
+ --data.transform.repack.textop-actions \
270
+ --data.transform.repack.action-chunk-size=$ACTION_CHUNK_SIZE \
271
+ --data.transform.repack.pad-action-dim=39 \
272
+ --data.transform.repack.pad-state-dim=39 \
273
+ --data.transform.field.action_norm_type=bounds_q99 \
274
+ --data.transform.field.stat-path=assets/stats/seed_all_g1_motion39_textop_stats.json \
275
+ --data.transform.field.normalize-state \
276
+ --data.transform.field.pad-action-dim=39 \
277
+ --data.transform.field.pad-state-dim=39 \
278
+ --data.transform.model.resize.size 240 320 \
279
+ --data.transform.model.center_crop.size 240 320 \
280
+ --data.transform.model.no-img-aug \
281
+ --data.transform.model.text-only \
282
+ --data.transform.model.audio-only \
283
+ --data.transform.model.video-only \
284
+ --data.transform.model.video-size-pixels=$VIDEO_SIZE_PIXELS \
285
+ --model.model_name_or_path=$PSI_PRETRAIN_BACKBONE_PATH \
286
+ --model.noise-scheduler=flow \
287
+ --model.n_conditions=0 \
288
+ --model.action-chunk-size=$ACTION_CHUNK_SIZE \
289
+ --model.action-dim=39 \
290
+ --model.action-exec-horizon=$ACTION_EXEC_HORIZON \
291
+ --model.observation-horizon=$STATE_HORIZON \
292
+ --model.odim=39 \
293
+ --model.view_feature_dim=2048 \
294
+ --model.hidden-dim=$ACTION_HIDDEN_DIM \
295
+ --model.num-blocks=$ACTION_NUM_BLOCKS \
296
+ --model.nhead=$ACTION_NHEADS \
297
+ --model.use-future-progress-embedding \
298
+ --model.future-progress-dim=$PROGRESS_DIM \
299
+ --model.tune-vlm \
300
+ --model.lang-backbone-lr=1e-5 \
301
+ --model.no-use_film \
302
+ --model.no-combined_temb \
303
+ --model.rtc \
304
+ --model.max-delay=11 \
305
+ --train.max-grad-value=1 \
306
+ --train.loss-group-weight.joint-abs=1.0 \
307
+ --train.loss-group-weight.root-delta-xy=3.0 \
308
+ --train.loss-group-weight.root-rot5-delta=10.0 \
309
+ --train.loss-group-weight.root-height=15.0 \
310
+ --train.loss-group-weight.foot-contact=0.1
311
+ "
312
+ torchrun --nproc_per_node=$NPROC_PER_NODE --master_port=29500 scripts/train.py \
313
+ $args "$@"
posttrain-v1-psi0-all-7task-tune-vlm-direct-dense-actionhead-perfect-resume.sh ADDED
@@ -0,0 +1,259 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/usr/bin/env bash
2
+
3
+ set -euo pipefail
4
+
5
+ ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/Psi0-yth-v21
6
+ RUNS_BASE=$ROOT/runs/posttrain_v1_trimodal
7
+ TARGET=${1:-${PSI_RESUME_TARGET:-}}
8
+
9
+ cd "$ROOT"
10
+
11
+ source /cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/envs/Psi0/bin/activate
12
+ [ -f .env ] && source .env
13
+ export WANDB_API_KEY="${WANDB_API_KEY:-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}"
14
+
15
+ export PYTHONPATH="$ROOT/src${PYTHONPATH:+:$PYTHONPATH}"
16
+ export CUDA_VISIBLE_DEVICES=${CUDA_VISIBLE_DEVICES:-0,1,2,3,4,5,6,7}
17
+ export OMP_NUM_THREADS=${OMP_NUM_THREADS:-8}
18
+ export TF_CPP_MIN_LOG_LEVEL=${TF_CPP_MIN_LOG_LEVEL:-3}
19
+ export MASTER_PORT=${MASTER_PORT:-29500}
20
+
21
+ count_visible_devices() {
22
+ local devices=$1
23
+ local n=0
24
+ IFS=',' read -r -a parts <<< "$devices"
25
+ for part in "${parts[@]}"; do
26
+ if [ -n "$part" ]; then
27
+ n=$((n + 1))
28
+ fi
29
+ done
30
+ echo "$n"
31
+ }
32
+
33
+ mapfile -t RESUME_INFO < <(
34
+ python3 - "$RUNS_BASE" "$TARGET" <<'PY'
35
+ import json
36
+ import os
37
+ import re
38
+ import sys
39
+
40
+ runs_base = os.path.abspath(sys.argv[1])
41
+ target = sys.argv[2].strip()
42
+
43
+
44
+ def load_json(path):
45
+ with open(path, "r") as f:
46
+ return json.load(f)
47
+
48
+
49
+ def is_run_dir(path):
50
+ return (
51
+ os.path.isfile(os.path.join(path, "run_config.json"))
52
+ and os.path.isfile(os.path.join(path, "argv.txt"))
53
+ and os.path.isdir(os.path.join(path, "checkpoints"))
54
+ )
55
+
56
+
57
+ def pick_latest_run():
58
+ candidates = []
59
+ if not os.path.isdir(runs_base):
60
+ raise SystemExit(f"Run base does not exist: {runs_base}")
61
+ for name in os.listdir(runs_base):
62
+ run_dir = os.path.join(runs_base, name)
63
+ if not os.path.isdir(run_dir) or not is_run_dir(run_dir):
64
+ continue
65
+ cfg = load_json(os.path.join(run_dir, "run_config.json"))
66
+ timestamp = cfg.get("timestamp")
67
+ try:
68
+ ts_key = int(str(timestamp))
69
+ except Exception:
70
+ ts_key = -1
71
+ candidates.append((ts_key, os.path.getmtime(run_dir), run_dir))
72
+ if not candidates:
73
+ raise SystemExit(f"No resumable run found under {runs_base}")
74
+ candidates.sort()
75
+ return candidates[-1][2]
76
+
77
+
78
+ def best_checkpoint_for_run(run_dir):
79
+ checkpoint_root = os.path.join(run_dir, "checkpoints")
80
+ candidates = []
81
+
82
+ best_dir = os.path.join(checkpoint_root, "best_ckpt")
83
+ best_info = os.path.join(checkpoint_root, "best_eval_metric.json")
84
+ if os.path.isfile(os.path.join(best_dir, "model.safetensors")) and os.path.isfile(best_info):
85
+ info = load_json(best_info)
86
+ candidates.append((int(info["global_step"]), best_dir, "best_ckpt"))
87
+
88
+ for name in os.listdir(checkpoint_root):
89
+ m = re.fullmatch(r"ckpt_(\d+)", name)
90
+ if not m:
91
+ continue
92
+ ckpt_dir = os.path.join(checkpoint_root, name)
93
+ if os.path.isfile(os.path.join(ckpt_dir, "model.safetensors")):
94
+ candidates.append((int(m.group(1)), ckpt_dir, name))
95
+
96
+ if not candidates:
97
+ raise SystemExit(f"No usable checkpoint found under {checkpoint_root}")
98
+
99
+ candidates.sort(key=lambda x: (x[0], x[2]))
100
+ return candidates[-1]
101
+
102
+
103
+ def resolve_target():
104
+ if not target:
105
+ run_dir = pick_latest_run()
106
+ step, ckpt_dir, _ = best_checkpoint_for_run(run_dir)
107
+ return run_dir, ckpt_dir, step
108
+
109
+ path = os.path.abspath(target)
110
+ if is_run_dir(path):
111
+ step, ckpt_dir, _ = best_checkpoint_for_run(path)
112
+ return path, ckpt_dir, step
113
+
114
+ if os.path.isfile(os.path.join(path, "model.safetensors")):
115
+ run_dir = os.path.dirname(os.path.dirname(path))
116
+ if not is_run_dir(run_dir):
117
+ raise SystemExit(f"Checkpoint parent is not a valid run dir: {run_dir}")
118
+
119
+ name = os.path.basename(path)
120
+ if name == "best_ckpt":
121
+ info = load_json(os.path.join(os.path.dirname(path), "best_eval_metric.json"))
122
+ step = int(info["global_step"])
123
+ else:
124
+ m = re.fullmatch(r"ckpt_(\d+)", name)
125
+ if not m:
126
+ raise SystemExit(f"Unsupported checkpoint dir name: {name}")
127
+ step = int(m.group(1))
128
+ return run_dir, path, step
129
+
130
+ raise SystemExit(
131
+ "Target must be empty, a run dir, or an exact checkpoint dir with model.safetensors"
132
+ )
133
+
134
+
135
+ run_dir, ckpt_dir, resume_step = resolve_target()
136
+ cfg = load_json(os.path.join(run_dir, "run_config.json"))
137
+ wandb_cfg = cfg.get("wandb") or {}
138
+ env_cfg = cfg.get("environment_variables") or {}
139
+
140
+ expected_world_size = env_cfg.get("WORLD_SIZE") or ""
141
+ if not expected_world_size:
142
+ m = re.search(r"\.gpus(\d+)\.", str(wandb_cfg.get("name") or ""))
143
+ if m:
144
+ expected_world_size = m.group(1)
145
+
146
+ print(run_dir)
147
+ print(ckpt_dir)
148
+ print(str(resume_step))
149
+ print(str(wandb_cfg.get("id") or ""))
150
+ print(str(cfg.get("timestamp") or ""))
151
+ print(str(expected_world_size or ""))
152
+ PY
153
+ )
154
+
155
+ RUN_DIR=${RESUME_INFO[0]}
156
+ RESUME_CKPT_DIR=${RESUME_INFO[1]}
157
+ RESUME_STEP=${RESUME_INFO[2]}
158
+ WANDB_RUN_ID=${RESUME_INFO[3]}
159
+ RUN_TIMESTAMP=${RESUME_INFO[4]}
160
+ EXPECTED_WORLD_SIZE=${RESUME_INFO[5]}
161
+
162
+ if [ -z "$WANDB_RUN_ID" ]; then
163
+ echo "Missing wandb run id in $RUN_DIR/run_config.json"
164
+ exit 1
165
+ fi
166
+
167
+ NPROC_PER_NODE=$(count_visible_devices "$CUDA_VISIBLE_DEVICES")
168
+ if [ "$NPROC_PER_NODE" -le 0 ]; then
169
+ echo "Invalid CUDA_VISIBLE_DEVICES: $CUDA_VISIBLE_DEVICES"
170
+ exit 1
171
+ fi
172
+
173
+ if [ -n "$EXPECTED_WORLD_SIZE" ] && [ "$NPROC_PER_NODE" != "$EXPECTED_WORLD_SIZE" ]; then
174
+ echo "World size mismatch: original run used $EXPECTED_WORLD_SIZE GPUs, current CUDA_VISIBLE_DEVICES=$CUDA_VISIBLE_DEVICES gives $NPROC_PER_NODE"
175
+ exit 1
176
+ fi
177
+
178
+ ARGV_FILE=$RUN_DIR/argv.txt
179
+ if [ ! -f "$ARGV_FILE" ]; then
180
+ echo "Missing argv.txt: $ARGV_FILE"
181
+ exit 1
182
+ fi
183
+
184
+ ORIG_ARGS=()
185
+ while IFS= read -r line || [ -n "$line" ]; do
186
+ [ -z "$line" ] && continue
187
+ read -r -a parts <<< "$line"
188
+ ORIG_ARGS+=("${parts[@]}")
189
+ done < "$ARGV_FILE"
190
+
191
+ SKIP_KEYS=(
192
+ --train.resume_from_checkpoint
193
+ --train.resume_load_full_state
194
+ --train.skip_resumed_steps
195
+ --wandb.id
196
+ --wandb.resume
197
+ )
198
+
199
+ FILTERED_ARGS=()
200
+ i=0
201
+ while [ "$i" -lt "${#ORIG_ARGS[@]}" ]; do
202
+ token=${ORIG_ARGS[$i]}
203
+ matched=0
204
+
205
+ for key in "${SKIP_KEYS[@]}"; do
206
+ if [[ "$token" == "$key" ]]; then
207
+ matched=1
208
+ i=$((i + 1))
209
+ while [ "$i" -lt "${#ORIG_ARGS[@]}" ] && [[ ! "${ORIG_ARGS[$i]}" == --* ]]; do
210
+ i=$((i + 1))
211
+ done
212
+ break
213
+ fi
214
+
215
+ if [[ "$token" == "$key="* ]]; then
216
+ matched=1
217
+ i=$((i + 1))
218
+ break
219
+ fi
220
+ done
221
+
222
+ if [ "$matched" -eq 0 ]; then
223
+ FILTERED_ARGS+=("$token")
224
+ i=$((i + 1))
225
+ fi
226
+ done
227
+
228
+ FINAL_ARGS=(
229
+ "${FILTERED_ARGS[@]}"
230
+ "--train.resume_from_checkpoint=$RESUME_CKPT_DIR"
231
+ "--train.resume_load_full_state"
232
+ "--wandb.id=$WANDB_RUN_ID"
233
+ "--wandb.resume=must"
234
+ )
235
+
236
+ export WANDB_RUN_ID
237
+ export WANDB_RESUME=must
238
+
239
+ echo "Perfect resume:"
240
+ echo " run_dir = $RUN_DIR"
241
+ echo " checkpoint = $RESUME_CKPT_DIR"
242
+ echo " resume_step = $RESUME_STEP"
243
+ echo " timestamp = $RUN_TIMESTAMP"
244
+ echo " wandb_id = $WANDB_RUN_ID"
245
+ echo " wandb_auth = configured"
246
+ echo " gpus = $CUDA_VISIBLE_DEVICES"
247
+ echo " nproc = $NPROC_PER_NODE"
248
+ echo " master_port = $MASTER_PORT"
249
+
250
+ if [ "${PSI_DRY_RUN:-0}" = "1" ]; then
251
+ echo " mode = dry-run"
252
+ exit 0
253
+ fi
254
+
255
+ exec torchrun \
256
+ --standalone \
257
+ --nproc_per_node="$NPROC_PER_NODE" \
258
+ --master_port="$MASTER_PORT" \
259
+ "${FINAL_ARGS[@]}"
posttrain-v1-psi0-all-7task-tune-vlm-direct-dense-full.sh ADDED
@@ -0,0 +1,421 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ pkill -f monitor.py
2
+
3
+ cd /cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/Psi0-yth-v21
4
+ export PYTHONPATH=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/Psi0-yth-v21/src
5
+ export CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7
6
+ export PSI_OUTPUT_DIR=${PSI_OUTPUT_DIR:-/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/Psi0-yth-v21/runs}
7
+ export PSI_LOG_REPORT_TO=wandb
8
+
9
+ source /cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/Psi0-yth-v15/.venv-psi/bin/activate
10
+ [ -f .env ] && source .env
11
+
12
+ export WANDB_API_KEY="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"
13
+
14
+ export OMP_NUM_THREADS=8
15
+ export TF_CPP_MIN_LOG_LEVEL=3
16
+
17
+ # Keep JSON sources readable in this script, but pass compact JSON to tyro.
18
+ compact_json() {
19
+ python -c 'import json, sys; print(json.dumps(json.load(sys.stdin), separators=(",", ":")))'
20
+ }
21
+
22
+ PSI_SEED_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/Psi0/dataset/seed_all
23
+ PSI_SEED_METADATA=metadata/seed_metadata_v004.csv
24
+ PSI_TEXT_RECORDS_CACHE_DIR=${PSI_TEXT_RECORDS_CACHE_DIR:-$PSI_SEED_ROOT/.index_cache_multimodal_yth}
25
+
26
+ PSI_DANCE_MOTION_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/GR00T-WholeBodyControl/gear_sonic_deploy/reference/converted/psi0_dance/gvhmr
27
+ PSI_DANCE_AUDIO_VIDEO_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/dance_1723_video_psi0/dance_1723_video_10s
28
+ PSI_DANCE_VIDEO_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/dance_1723_video_psi0/dance_1723_video_10s
29
+ PSI_DANCE_VIDEO_EMBED_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/dance_1723_video_psi0/dance_1723_video_10s_embeds
30
+ PSI_VIDEO_DEMO_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/videogen/motion_example_0506_demo
31
+ PSI_2264_WITH_AUDIO_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/videogen/2264_with_audio
32
+ PSI_QWEN3_TTS_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/qwen3-tts
33
+ PSI_VIDEO_CLIP_DATASET_ROOT=${PSI_VIDEO_CLIP_DATASET_ROOT:-/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/videogen/psi0_video_clip_50f_dense_stride1_direct_full}
34
+ PSI_DANCE_VIDEO_CLIP_ROOT=$PSI_VIDEO_CLIP_DATASET_ROOT/dance_1723_video_10s
35
+ PSI_MOTION_EXAMPLE_VIDEO_CLIP_ROOT=$PSI_VIDEO_CLIP_DATASET_ROOT/motion_example_0506_demo
36
+ PSI_2264_WITH_AUDIO_VIDEO_CLIP_ROOT=$PSI_VIDEO_CLIP_DATASET_ROOT/2264_with_audio
37
+ PSI_DANCE_VIDEO_CLIP_EMBED_ROOT=${PSI_DANCE_VIDEO_CLIP_EMBED_ROOT:-$PSI_DANCE_VIDEO_CLIP_ROOT/embeds}
38
+ PSI_MOTION_EXAMPLE_VIDEO_CLIP_EMBED_ROOT=${PSI_MOTION_EXAMPLE_VIDEO_CLIP_EMBED_ROOT:-$PSI_MOTION_EXAMPLE_VIDEO_CLIP_ROOT/embeds}
39
+ PSI_2264_WITH_AUDIO_VIDEO_CLIP_EMBED_ROOT=${PSI_2264_WITH_AUDIO_VIDEO_CLIP_EMBED_ROOT:-$PSI_2264_WITH_AUDIO_VIDEO_CLIP_ROOT/embeds}
40
+ PSI_MOTION_EXAMPLE_VIDEO_EMBED_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/videogen/motion_example_0506_demo_video_embeds
41
+ PSI_AUDIO_RECORDS_CACHE_DIR=${PSI_AUDIO_RECORDS_CACHE_DIR:-$PSI_2264_WITH_AUDIO_ROOT/.index_cache_multimodal_yth}
42
+ PSI_AUDIO_COMMAND_RECORDS_CACHE_DIR=${PSI_AUDIO_COMMAND_RECORDS_CACHE_DIR:-$PSI_QWEN3_TTS_ROOT/.index_cache_multimodal_yth}
43
+ PSI_DANCE_AUDIO_NPY_ROOT=${PSI_DANCE_AUDIO_NPY_ROOT:-$PSI_DANCE_AUDIO_VIDEO_ROOT/audio_npy_yth}
44
+ PSI_DANCE_AUDIO_NPY_MANIFEST=${PSI_DANCE_AUDIO_NPY_MANIFEST:-$PSI_DANCE_AUDIO_VIDEO_ROOT/metadata_audio_npy.jsonl}
45
+ PSI_2264_AUDIO_NPY_ROOT=${PSI_2264_AUDIO_NPY_ROOT:-$PSI_2264_WITH_AUDIO_ROOT/audio_npy_yth}
46
+ PSI_2264_AUDIO_NPY_MANIFEST=${PSI_2264_AUDIO_NPY_MANIFEST:-$PSI_2264_WITH_AUDIO_ROOT/metadata_audio_npy.jsonl}
47
+ PSI_QWEN3_TTS_AUDIO_NPY_ROOT=${PSI_QWEN3_TTS_AUDIO_NPY_ROOT:-$PSI_QWEN3_TTS_ROOT/audio_npy_yth}
48
+ PSI_QWEN3_TTS_AUDIO_NPY_MANIFEST=${PSI_QWEN3_TTS_AUDIO_NPY_MANIFEST:-$PSI_QWEN3_TTS_ROOT/metadata_audio_npy.jsonl}
49
+
50
+ if [ -f "$PSI_2264_AUDIO_NPY_MANIFEST" ]; then
51
+ PSI_2264_AUDIO_METADATA_JSONL=$PSI_2264_AUDIO_NPY_MANIFEST
52
+ PSI_2264_AUDIO_PATH_KEY=audio_path
53
+ else
54
+ PSI_2264_AUDIO_METADATA_JSONL=metadata_convert.jsonl
55
+ PSI_2264_AUDIO_PATH_KEY=video_path
56
+ fi
57
+
58
+ if [ -f "$PSI_DANCE_AUDIO_NPY_MANIFEST" ]; then
59
+ PSI_DANCE_AUDIO_SOURCE_JSON=$(cat <<JSON
60
+ {
61
+ "name": "psi0_dance_audio",
62
+ "dataset_root": "$PSI_DANCE_AUDIO_VIDEO_ROOT",
63
+ "metadata_jsonl": "$PSI_DANCE_AUDIO_NPY_MANIFEST",
64
+ "motion_dir_key": "converted_path",
65
+ "audio_path_key": "audio_path",
66
+ "records_cache_dir": "$PSI_AUDIO_RECORDS_CACHE_DIR",
67
+ "dataset_name": "seed-audio-g1"
68
+ }
69
+ JSON
70
+ )
71
+ else
72
+ PSI_DANCE_AUDIO_SOURCE_JSON=$(cat <<JSON
73
+ {
74
+ "name": "psi0_dance_audio",
75
+ "root_dir": "$PSI_DANCE_MOTION_ROOT",
76
+ "video_dir": "$PSI_DANCE_AUDIO_VIDEO_ROOT",
77
+ "records_cache_dir": "$PSI_AUDIO_RECORDS_CACHE_DIR",
78
+ "dataset_name": "seed-audio-g1"
79
+ }
80
+ JSON
81
+ )
82
+ fi
83
+
84
+ if [ -z "$PSI_AUDIO_SOURCES_JSON" ]; then
85
+ PSI_AUDIO_SOURCES_JSON=$(compact_json <<JSON
86
+ [
87
+ $PSI_DANCE_AUDIO_SOURCE_JSON,
88
+ {
89
+ "name": "2264_with_audio",
90
+ "dataset_root": "$PSI_2264_WITH_AUDIO_ROOT",
91
+ "metadata_jsonl": "$PSI_2264_AUDIO_METADATA_JSONL",
92
+ "motion_dir_key": "converted_path",
93
+ "audio_path_key": "$PSI_2264_AUDIO_PATH_KEY",
94
+ "records_cache_dir": "$PSI_AUDIO_RECORDS_CACHE_DIR",
95
+ "dataset_name": "2264-with-audio-g1"
96
+ }
97
+ ]
98
+ JSON
99
+ )
100
+ fi
101
+
102
+ if [ -f "$PSI_QWEN3_TTS_AUDIO_NPY_MANIFEST" ]; then
103
+ PSI_AUDIO_COMMAND_SOURCE_JSON=$(cat <<JSON
104
+ {
105
+ "name": "qwen3_tts_seed_audio_command",
106
+ "root_dir": "$PSI_SEED_ROOT",
107
+ "dataset_root": "$PSI_QWEN3_TTS_ROOT",
108
+ "metadata_jsonl": "$PSI_QWEN3_TTS_AUDIO_NPY_MANIFEST",
109
+ "motion_dir_key": "filename",
110
+ "audio_path_key": "audio_path",
111
+ "instruction_key": "content_short_description",
112
+ "motion_metadata_csv": "$PSI_SEED_METADATA",
113
+ "motion_metadata_key": "filename",
114
+ "motion_metadata_path_key": "move_g1_path",
115
+ "audio_file_extensions": ".npy",
116
+ "records_cache_dir": "$PSI_AUDIO_COMMAND_RECORDS_CACHE_DIR",
117
+ "audio_task": "command",
118
+ "dataset_name": "seed-audio-command-g1"
119
+ }
120
+ JSON
121
+ )
122
+ else
123
+ PSI_AUDIO_COMMAND_SOURCE_JSON=$(cat <<JSON
124
+ {
125
+ "name": "qwen3_tts_seed_audio_command",
126
+ "root_dir": "$PSI_SEED_ROOT",
127
+ "dataset_root": "$PSI_QWEN3_TTS_ROOT",
128
+ "metadata_jsonl": "metadata.jsonl",
129
+ "motion_dir_key": "filename",
130
+ "audio_path_key": "audio_path",
131
+ "instruction_key": "content_short_description",
132
+ "motion_metadata_csv": "$PSI_SEED_METADATA",
133
+ "motion_metadata_key": "filename",
134
+ "motion_metadata_path_key": "move_g1_path",
135
+ "audio_file_extensions": ".wav",
136
+ "records_cache_dir": "$PSI_AUDIO_COMMAND_RECORDS_CACHE_DIR",
137
+ "audio_task": "command",
138
+ "dataset_name": "seed-audio-command-g1"
139
+ }
140
+ JSON
141
+ )
142
+ fi
143
+
144
+ if [ -z "$PSI_AUDIO_COMMAND_SOURCES_JSON" ]; then
145
+ PSI_AUDIO_COMMAND_SOURCES_JSON=$(compact_json <<JSON
146
+ [
147
+ $PSI_AUDIO_COMMAND_SOURCE_JSON
148
+ ]
149
+ JSON
150
+ )
151
+ fi
152
+
153
+ if [ -z "$PSI_VIDEO_SOURCES_JSON" ]; then
154
+ PSI_VIDEO_SOURCES_JSON=$(compact_json <<JSON
155
+ [
156
+ {
157
+ "name": "psi0_dance_gvhmr",
158
+ "root_dir": "$PSI_DANCE_MOTION_ROOT",
159
+ "video_dir": "$PSI_DANCE_VIDEO_ROOT",
160
+ "video_embed_dir": "$PSI_DANCE_VIDEO_EMBED_ROOT",
161
+ "dataset_name": "seed-video-g1"
162
+ },
163
+ {
164
+ "name": "motion_example_0506_demo",
165
+ "dataset_root": "$PSI_VIDEO_DEMO_ROOT",
166
+ "metadata_jsonl": "metadata.jsonl",
167
+ "motion_dir_key": "converted_path",
168
+ "video_path_key": "video_path",
169
+ "instruction_key": "video_caption",
170
+ "video_embed_dir": "$PSI_MOTION_EXAMPLE_VIDEO_EMBED_ROOT",
171
+ "dataset_name": "motion-example-video-g1"
172
+ }
173
+ ]
174
+ JSON
175
+ )
176
+ fi
177
+
178
+ if [ -z "$PSI_VIDEO_CLIP_SOURCES_JSON" ]; then
179
+ PSI_VIDEO_CLIP_SOURCES_JSON=$(compact_json <<JSON
180
+ [
181
+ {
182
+ "name": "dance_1723_video_10s_clip50",
183
+ "dataset_root": "$PSI_DANCE_VIDEO_CLIP_ROOT",
184
+ "metadata_jsonl": "metadata.jsonl",
185
+ "motion_dir_key": "converted_path",
186
+ "video_path_key": "video_path",
187
+ "instruction_key": "video_caption",
188
+ "video_embed_dir": "$PSI_DANCE_VIDEO_CLIP_EMBED_ROOT",
189
+ "dataset_name": "dance-video-clip-g1"
190
+ },
191
+ {
192
+ "name": "motion_example_0506_demo_clip50",
193
+ "dataset_root": "$PSI_MOTION_EXAMPLE_VIDEO_CLIP_ROOT",
194
+ "metadata_jsonl": "metadata.jsonl",
195
+ "motion_dir_key": "converted_path",
196
+ "video_path_key": "video_path",
197
+ "instruction_key": "video_caption",
198
+ "video_embed_dir": "$PSI_MOTION_EXAMPLE_VIDEO_CLIP_EMBED_ROOT",
199
+ "dataset_name": "motion-example-video-clip-g1"
200
+ },
201
+ {
202
+ "name": "2264_with_audio_clip50",
203
+ "dataset_root": "$PSI_2264_WITH_AUDIO_VIDEO_CLIP_ROOT",
204
+ "metadata_jsonl": "metadata.jsonl",
205
+ "motion_dir_key": "converted_path",
206
+ "video_path_key": "video_path",
207
+ "instruction_key": "video_caption",
208
+ "video_embed_dir": "$PSI_2264_WITH_AUDIO_VIDEO_CLIP_EMBED_ROOT",
209
+ "dataset_name": "2264-with-audio-video-clip-g1"
210
+ }
211
+ ]
212
+ JSON
213
+ )
214
+ fi
215
+
216
+ # 7-task from-scratch mix, excluding legacy VIDEO_TEXT / VIDEO_ONLY.
217
+ # Smaller data sources get larger weights. These use a tempered inverse-size
218
+ # rule, (47M / source_size)^0.25, so rare modalities are upweighted without
219
+ # completely starving text/video_clip in BatchMixtureSampler.
220
+ # text stride1 frame samples: ~47,000,000
221
+ # music audio dance frame samples: 213,731
222
+ # audio_command wav files: 41,760
223
+ # dense video_clip metadata: 4,597,576
224
+ PSI_TEXT_TASK_WEIGHT=1
225
+ PSI_AUDIO_TEXT_TASK_WEIGHT=1
226
+ PSI_AUDIO_ONLY_TASK_WEIGHT=1
227
+ PSI_AUDIO_COMMAND_TEXT_TASK_WEIGHT=1
228
+ PSI_AUDIO_COMMAND_ONLY_TASK_WEIGHT=1
229
+ PSI_VIDEO_TEXT_TASK_WEIGHT=0
230
+ PSI_VIDEO_ONLY_TASK_WEIGHT=0
231
+ PSI_VIDEO_CLIP_TEXT_TASK_WEIGHT=1
232
+ PSI_VIDEO_CLIP_ONLY_TASK_WEIGHT=1
233
+
234
+ PSI_PRETRAIN_BACKBONE_PATH=/cpfs01/VideoGen/public_ckpt/Qwen/Qwen2.5-Omni-3B
235
+ NUM_PAST_FRAMES=0
236
+ STATE_HORIZON=$((NUM_PAST_FRAMES + 1))
237
+ ACTION_CHUNK_SIZE=30
238
+ ACTION_EXEC_HORIZON=20
239
+ PROGRESS_DIM=5
240
+ VIDEO_SAMPLE_FPS=1.0
241
+ VIDEO_CLIP_SAMPLE_FPS=8.0
242
+ VIDEO_SIZE_PIXELS=50176
243
+ VIDEO_DECODER=ffmpeg
244
+ VIDEO_CACHE_SIZE=64
245
+ VIDEO_CLIP_MOTION_CACHE_SIZE=256
246
+ VIDEO_RECORD_BUILD_WORKERS=${VIDEO_RECORD_BUILD_WORKERS:-32}
247
+ VIDEO_RECORD_BUILD_CHUNK_SIZE=${VIDEO_RECORD_BUILD_CHUNK_SIZE:-4096}
248
+ VIDEO_CLIP_TRUST_METADATA=${VIDEO_CLIP_TRUST_METADATA:-1}
249
+ VIDEO_CLIP_RECORDS_CACHE_DIR=${VIDEO_CLIP_RECORDS_CACHE_DIR:-$PSI_VIDEO_CLIP_DATASET_ROOT/.index_cache_multimodal_yth}
250
+ if [ "$VIDEO_CLIP_TRUST_METADATA" = "1" ] || [ "$VIDEO_CLIP_TRUST_METADATA" = "true" ] || [ "$VIDEO_CLIP_TRUST_METADATA" = "True" ]; then
251
+ VIDEO_CLIP_TRUST_METADATA_FLAG="--data.video-clip.trust-metadata"
252
+ else
253
+ VIDEO_CLIP_TRUST_METADATA_FLAG="--data.video-clip.no-trust-metadata"
254
+ fi
255
+
256
+ # DATALOADER_NUM_WORKERS=4
257
+ # VAL_DATALOADER_NUM_WORKERS=2
258
+
259
+ # DATALOADER_NUM_WORKERS=${DATALOADER_NUM_WORKERS:-0}
260
+ # VAL_DATALOADER_NUM_WORKERS=${VAL_DATALOADER_NUM_WORKERS:-0}
261
+
262
+ DATALOADER_NUM_WORKERS=${DATALOADER_NUM_WORKERS:-2}
263
+ VAL_DATALOADER_NUM_WORKERS=${VAL_DATALOADER_NUM_WORKERS:-2}
264
+
265
+ NPROC_PER_NODE=$(echo $CUDA_VISIBLE_DEVICES | tr ',' '\n' | wc -l)
266
+
267
+ echo "Training with $NPROC_PER_NODE GPU(s): $CUDA_VISIBLE_DEVICES"
268
+ echo "Text root: $PSI_SEED_ROOT"
269
+ echo "Text records cache dir: $PSI_TEXT_RECORDS_CACHE_DIR"
270
+ echo "Dance motion root: $PSI_DANCE_MOTION_ROOT"
271
+ echo "Dance audio/video root: $PSI_DANCE_AUDIO_VIDEO_ROOT"
272
+ echo "Dance video root: $PSI_DANCE_VIDEO_ROOT"
273
+ echo "Dance video embed root: $PSI_DANCE_VIDEO_EMBED_ROOT"
274
+ echo "2264 with audio root: $PSI_2264_WITH_AUDIO_ROOT"
275
+ echo "Dance audio npy root: $PSI_DANCE_AUDIO_NPY_ROOT"
276
+ echo "Dance audio manifest: $PSI_DANCE_AUDIO_NPY_MANIFEST"
277
+ echo "2264 audio npy root: $PSI_2264_AUDIO_NPY_ROOT"
278
+ echo "2264 audio manifest: $PSI_2264_AUDIO_METADATA_JSONL"
279
+ echo "Audio records cache dir: $PSI_AUDIO_RECORDS_CACHE_DIR"
280
+ echo "Audio command records cache dir: $PSI_AUDIO_COMMAND_RECORDS_CACHE_DIR"
281
+ echo "Qwen3-TTS acdudio command root: $PSI_QWEN3_TTS_ROOT"
282
+ echo "Qwen3-TTS audio npy root: $PSI_QWEN3_TTS_AUDIO_NPY_ROOT"
283
+ echo "Qwen3-TTS audio manifest: $PSI_QWEN3_TTS_AUDIO_NPY_MANIFEST"
284
+ echo "Audio sources: $PSI_AUDIO_SOURCES_JSON"
285
+ echo "Audio command sources: $PSI_AUDIO_COMMAND_SOURCES_JSON"
286
+ echo "Video sources: $PSI_VIDEO_SOURCES_JSON"
287
+ echo "Video clip sources: $PSI_VIDEO_CLIP_SOURCES_JSON"
288
+ echo "DataLoader workers: train=$DATALOADER_NUM_WORKERS val=$VAL_DATALOADER_NUM_WORKERS"
289
+ echo "Sampler: task_mixture"
290
+ echo "Task weights: text=$PSI_TEXT_TASK_WEIGHT audio_text=$PSI_AUDIO_TEXT_TASK_WEIGHT audio_only=$PSI_AUDIO_ONLY_TASK_WEIGHT audio_command_text=$PSI_AUDIO_COMMAND_TEXT_TASK_WEIGHT audio_command_only=$PSI_AUDIO_COMMAND_ONLY_TASK_WEIGHT video_text=$PSI_VIDEO_TEXT_TASK_WEIGHT video_only=$PSI_VIDEO_ONLY_TASK_WEIGHT video_clip_text=$PSI_VIDEO_CLIP_TEXT_TASK_WEIGHT video_clip_only=$PSI_VIDEO_CLIP_ONLY_TASK_WEIGHT"
291
+ # echo "Resume run dir: $PSI_PRETRAIN_RUN_DIR"
292
+ echo "TextOp 39D action/state enabled"
293
+ echo "Future progress embedding: enabled, dim $PROGRESS_DIM"
294
+ echo "Backbone path: $PSI_PRETRAIN_BACKBONE_PATH"
295
+ echo "Output dir: $PSI_OUTPUT_DIR"
296
+ echo "Video clip trust metadata: $VIDEO_CLIP_TRUST_METADATA"
297
+ echo "Video clip records cache dir: $VIDEO_CLIP_RECORDS_CACHE_DIR"
298
+ echo "Video record build workers: $VIDEO_RECORD_BUILD_WORKERS"
299
+ echo "Video record build chunk size: $VIDEO_RECORD_BUILD_CHUNK_SIZE"
300
+
301
+ args="posttrain_v1_seed_text_audio_video_mix_psi0_config \
302
+ --seed=292285 \
303
+ --exp=posttrain \
304
+ --timestamp=$(date +"%y%m%d%H%M") \
305
+ --train.name=posttrain_v1_trimodal \
306
+ --train.data_parallel=ddp \
307
+ --train.mixed_precision=bf16 \
308
+ --train.train_batch_size=64 \
309
+ --train.val_batch_size=64 \
310
+ --train.num-workers=$DATALOADER_NUM_WORKERS \
311
+ --train.val-num-workers=$VAL_DATALOADER_NUM_WORKERS \
312
+ --train.resume_from_checkpoint=None \
313
+ --train.max_checkpoints_to_keep=1 \
314
+ --train.gradient_accumulation_steps=1 \
315
+ --train.learning_rate=1e-4 \
316
+ --train.max_training_steps=500000 \
317
+ --train.warmup_ratio=None \
318
+ --train.warmup_steps=1000 \
319
+ --train.checkpointing_steps=1000 \
320
+ --train.validation_steps=1000 \
321
+ --train.val_num_batches=20 \
322
+ --train.max_grad_norm=1.0 \
323
+ --train.lr_scheduler_type=cosine_with_min_lr \
324
+ --train.lr_scheduler_kwargs.betas 0.9 0.999 \
325
+ --train.lr_scheduler_kwargs.weight_decay=0 \
326
+ --train.lr_scheduler_kwargs.eps=1e-8 \
327
+ --log.report_to=$PSI_LOG_REPORT_TO \
328
+ --train.output_dir=$PSI_OUTPUT_DIR \
329
+ --data.text.root-dir=$PSI_SEED_ROOT \
330
+ --data.text.metadata-csv=$PSI_SEED_METADATA \
331
+ --data.text.records-cache-dir=$PSI_TEXT_RECORDS_CACHE_DIR \
332
+ --data.text-ratio=$PSI_TEXT_TASK_WEIGHT \
333
+ --data.audio.sources-json=$PSI_AUDIO_SOURCES_JSON \
334
+ --data.audio-ratio=$PSI_AUDIO_TEXT_TASK_WEIGHT \
335
+ --data.audio-only-ratio=$PSI_AUDIO_ONLY_TASK_WEIGHT \
336
+ --data.audio.sample-stride=1 \
337
+ --data.audio.motion-fps=50 \
338
+ --data.audio.audio-sampling-rate=16000 \
339
+ --data.audio.audio-window-seconds=1.0 \
340
+ --data.audio.audio-left-context-seconds=0.0 \
341
+ --data.audio.full-audio \
342
+ --data.audio-command.sources-json=$PSI_AUDIO_COMMAND_SOURCES_JSON \
343
+ --data.audio-command-ratio=$PSI_AUDIO_COMMAND_TEXT_TASK_WEIGHT \
344
+ --data.audio-command-only-ratio=$PSI_AUDIO_COMMAND_ONLY_TASK_WEIGHT \
345
+ --data.audio-command.sample-stride=1 \
346
+ --data.audio-command.motion-fps=50 \
347
+ --data.audio-command.audio-sampling-rate=16000 \
348
+ --data.audio-command.audio-window-seconds=1.0 \
349
+ --data.audio-command.audio-left-context-seconds=0.0 \
350
+ --data.audio-command.full-audio \
351
+ --data.video-ratio=$PSI_VIDEO_TEXT_TASK_WEIGHT \
352
+ --data.video-only-ratio=$PSI_VIDEO_ONLY_TASK_WEIGHT \
353
+ --data.video.sample-stride=1 \
354
+ --data.video.motion-fps=50 \
355
+ --data.video.video-sample-fps=$VIDEO_SAMPLE_FPS \
356
+ --data.video.video-size-pixels=$VIDEO_SIZE_PIXELS \
357
+ --data.video.video-decoder=$VIDEO_DECODER \
358
+ --data.video.video-cache-size=$VIDEO_CACHE_SIZE \
359
+ --data.video.record-build-workers=$VIDEO_RECORD_BUILD_WORKERS \
360
+ --data.video.record-build-chunk-size=$VIDEO_RECORD_BUILD_CHUNK_SIZE \
361
+ --data.video.sources-json=$PSI_VIDEO_SOURCES_JSON \
362
+ --data.video-clip-text-ratio=$PSI_VIDEO_CLIP_TEXT_TASK_WEIGHT \
363
+ --data.video-clip-only-ratio=$PSI_VIDEO_CLIP_ONLY_TASK_WEIGHT \
364
+ --data.video-clip.sources-json=$PSI_VIDEO_CLIP_SOURCES_JSON \
365
+ --data.video-clip.sample-stride=1 \
366
+ --data.video-clip.motion-fps=50 \
367
+ --data.video-clip.video-sample-fps=$VIDEO_CLIP_SAMPLE_FPS \
368
+ --data.video-clip.video-size-pixels=$VIDEO_SIZE_PIXELS \
369
+ --data.video-clip.video-decoder=$VIDEO_DECODER \
370
+ --data.video-clip.video-cache-size=$VIDEO_CACHE_SIZE \
371
+ --data.video-clip.motion-cache-size=$VIDEO_CLIP_MOTION_CACHE_SIZE \
372
+ --data.video-clip.record-build-workers=$VIDEO_RECORD_BUILD_WORKERS \
373
+ --data.video-clip.record-build-chunk-size=$VIDEO_RECORD_BUILD_CHUNK_SIZE \
374
+ $VIDEO_CLIP_TRUST_METADATA_FLAG \
375
+ --data.video-clip.records-cache-dir=$VIDEO_CLIP_RECORDS_CACHE_DIR \
376
+ --data.video-clip.action-exec-horizon=$ACTION_EXEC_HORIZON \
377
+ --data.sampler=task_mixture \
378
+ --data.val-task=text,audio,audio_only,audio_command,audio_command_only,video_clip,video_clip_only \
379
+ --data.transform.repack.num-past-frames=$NUM_PAST_FRAMES \
380
+ --data.transform.repack.textop-actions \
381
+ --data.transform.repack.action-chunk-size=$ACTION_CHUNK_SIZE \
382
+ --data.transform.repack.pad-action-dim=39 \
383
+ --data.transform.repack.pad-state-dim=39 \
384
+ --data.transform.field.action_norm_type=bounds_q99 \
385
+ --data.transform.field.stat-path=assets/stats/seed_all_g1_motion39_textop_stats.json \
386
+ --data.transform.field.normalize-state \
387
+ --data.transform.field.pad-action-dim=39 \
388
+ --data.transform.field.pad-state-dim=39 \
389
+ --data.transform.model.resize.size 240 320 \
390
+ --data.transform.model.center_crop.size 240 320 \
391
+ --data.transform.model.no-img-aug \
392
+ --data.transform.model.text-only \
393
+ --data.transform.model.audio-only \
394
+ --data.transform.model.video-only \
395
+ --data.transform.model.video-size-pixels=$VIDEO_SIZE_PIXELS \
396
+ --model.model_name_or_path=$PSI_PRETRAIN_BACKBONE_PATH \
397
+ --model.noise-scheduler=flow \
398
+ --model.n_conditions=0 \
399
+ --model.action-chunk-size=$ACTION_CHUNK_SIZE \
400
+ --model.action-dim=39 \
401
+ --model.action-exec-horizon=$ACTION_EXEC_HORIZON \
402
+ --model.observation-horizon=$STATE_HORIZON \
403
+ --model.odim=39 \
404
+ --model.view_feature_dim=2048 \
405
+ --model.use-future-progress-embedding \
406
+ --model.future-progress-dim=$PROGRESS_DIM \
407
+ --model.tune-vlm \
408
+ --model.lang-backbone-lr=1e-5 \
409
+ --model.no-use_film \
410
+ --model.no-combined_temb \
411
+ --model.rtc \
412
+ --model.max-delay=11 \
413
+ --train.max-grad-value=1 \
414
+ --train.loss-group-weight.joint-abs=1.0 \
415
+ --train.loss-group-weight.root-delta-xy=3.0 \
416
+ --train.loss-group-weight.root-rot5-delta=10.0 \
417
+ --train.loss-group-weight.root-height=15.0 \
418
+ --train.loss-group-weight.foot-contact=0.1
419
+ "
420
+ torchrun --nproc_per_node=$NPROC_PER_NODE --master_port=29500 scripts/train.py \
421
+ $args "$@"
posttrain-v1-psi0-all-7task-tune-vlm-direct-dense.sh ADDED
@@ -0,0 +1,305 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ pkill -f monitor.py
2
+
3
+ cd /cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/Psi0-yth-v21
4
+ export PYTHONPATH=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/Psi0-yth-v21/src
5
+ export CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7
6
+ export PSI_OUTPUT_DIR=${PSI_OUTPUT_DIR:-/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/Psi0-yth-v21/runs}
7
+ export PSI_LOG_REPORT_TO=wandb
8
+
9
+ source /cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/Psi0-yth-v15/.venv-psi/bin/activate
10
+ [ -f .env ] && source .env
11
+
12
+ export WANDB_API_KEY="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"
13
+
14
+ export OMP_NUM_THREADS=8
15
+ export TF_CPP_MIN_LOG_LEVEL=3
16
+
17
+ # Keep JSON sources readable in this script, but pass compact JSON to tyro.
18
+ compact_json() {
19
+ python -c 'import json, sys; print(json.dumps(json.load(sys.stdin), separators=(",", ":")))'
20
+ }
21
+
22
+ PSI_SEED_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/Psi0/dataset/seed_all
23
+ PSI_SEED_METADATA=metadata/seed_metadata_v004.csv
24
+
25
+ PSI_DANCE_MOTION_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/GR00T-WholeBodyControl/gear_sonic_deploy/reference/converted/psi0_dance/gvhmr
26
+ PSI_DANCE_AUDIO_VIDEO_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/dance_1723_video_psi0/dance_1723_video_10s
27
+ PSI_DANCE_VIDEO_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/dance_1723_video_psi0/dance_1723_video_10s
28
+ PSI_DANCE_VIDEO_EMBED_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/dance_1723_video_psi0/dance_1723_video_10s_embeds
29
+ PSI_VIDEO_DEMO_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/videogen/motion_example_0506_demo
30
+ PSI_QWEN3_TTS_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/qwen3-tts
31
+ PSI_VIDEO_CLIP_DATASET_ROOT=${PSI_VIDEO_CLIP_DATASET_ROOT:-/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/videogen/psi0_video_clip_50f_dense_stride1_direct_full}
32
+ PSI_DANCE_VIDEO_CLIP_ROOT=$PSI_VIDEO_CLIP_DATASET_ROOT/dance_1723_video_10s
33
+ PSI_MOTION_EXAMPLE_VIDEO_CLIP_ROOT=$PSI_VIDEO_CLIP_DATASET_ROOT/motion_example_0506_demo
34
+ PSI_DANCE_VIDEO_CLIP_EMBED_ROOT=${PSI_DANCE_VIDEO_CLIP_EMBED_ROOT:-$PSI_DANCE_VIDEO_CLIP_ROOT/embeds}
35
+ PSI_MOTION_EXAMPLE_VIDEO_CLIP_EMBED_ROOT=${PSI_MOTION_EXAMPLE_VIDEO_CLIP_EMBED_ROOT:-$PSI_MOTION_EXAMPLE_VIDEO_CLIP_ROOT/embeds}
36
+ PSI_MOTION_EXAMPLE_VIDEO_EMBED_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/videogen/motion_example_0506_demo_video_embeds
37
+
38
+ if [ -z "$PSI_AUDIO_SOURCES_JSON" ]; then
39
+ PSI_AUDIO_SOURCES_JSON=$(compact_json <<JSON
40
+ [
41
+ {
42
+ "name": "psi0_dance_audio",
43
+ "root_dir": "$PSI_DANCE_MOTION_ROOT",
44
+ "video_dir": "$PSI_DANCE_AUDIO_VIDEO_ROOT",
45
+ "dataset_name": "seed-audio-g1"
46
+ }
47
+ ]
48
+ JSON
49
+ )
50
+ fi
51
+
52
+ if [ -z "$PSI_AUDIO_COMMAND_SOURCES_JSON" ]; then
53
+ PSI_AUDIO_COMMAND_SOURCES_JSON=$(compact_json <<JSON
54
+ [
55
+ {
56
+ "name": "qwen3_tts_seed_audio_command",
57
+ "root_dir": "$PSI_SEED_ROOT",
58
+ "dataset_root": "$PSI_QWEN3_TTS_ROOT",
59
+ "metadata_jsonl": "metadata.jsonl",
60
+ "motion_dir_key": "filename",
61
+ "audio_path_key": "audio_path",
62
+ "instruction_key": "content_short_description",
63
+ "motion_metadata_csv": "$PSI_SEED_METADATA",
64
+ "motion_metadata_key": "filename",
65
+ "motion_metadata_path_key": "move_g1_path",
66
+ "audio_file_extensions": ".wav",
67
+ "audio_task": "command",
68
+ "dataset_name": "seed-audio-command-g1"
69
+ }
70
+ ]
71
+ JSON
72
+ )
73
+ fi
74
+
75
+ if [ -z "$PSI_VIDEO_SOURCES_JSON" ]; then
76
+ PSI_VIDEO_SOURCES_JSON=$(compact_json <<JSON
77
+ [
78
+ {
79
+ "name": "psi0_dance_gvhmr",
80
+ "root_dir": "$PSI_DANCE_MOTION_ROOT",
81
+ "video_dir": "$PSI_DANCE_VIDEO_ROOT",
82
+ "video_embed_dir": "$PSI_DANCE_VIDEO_EMBED_ROOT",
83
+ "dataset_name": "seed-video-g1"
84
+ },
85
+ {
86
+ "name": "motion_example_0506_demo",
87
+ "dataset_root": "$PSI_VIDEO_DEMO_ROOT",
88
+ "metadata_jsonl": "metadata.jsonl",
89
+ "motion_dir_key": "converted_path",
90
+ "video_path_key": "video_path",
91
+ "instruction_key": "video_caption",
92
+ "video_embed_dir": "$PSI_MOTION_EXAMPLE_VIDEO_EMBED_ROOT",
93
+ "dataset_name": "motion-example-video-g1"
94
+ }
95
+ ]
96
+ JSON
97
+ )
98
+ fi
99
+
100
+ if [ -z "$PSI_VIDEO_CLIP_SOURCES_JSON" ]; then
101
+ PSI_VIDEO_CLIP_SOURCES_JSON=$(compact_json <<JSON
102
+ [
103
+ {
104
+ "name": "dance_1723_video_10s_clip50",
105
+ "dataset_root": "$PSI_DANCE_VIDEO_CLIP_ROOT",
106
+ "metadata_jsonl": "metadata.jsonl",
107
+ "motion_dir_key": "converted_path",
108
+ "video_path_key": "video_path",
109
+ "instruction_key": "video_caption",
110
+ "video_embed_dir": "$PSI_DANCE_VIDEO_CLIP_EMBED_ROOT",
111
+ "dataset_name": "dance-video-clip-g1"
112
+ },
113
+ {
114
+ "name": "motion_example_0506_demo_clip50",
115
+ "dataset_root": "$PSI_MOTION_EXAMPLE_VIDEO_CLIP_ROOT",
116
+ "metadata_jsonl": "metadata.jsonl",
117
+ "motion_dir_key": "converted_path",
118
+ "video_path_key": "video_path",
119
+ "instruction_key": "video_caption",
120
+ "video_embed_dir": "$PSI_MOTION_EXAMPLE_VIDEO_CLIP_EMBED_ROOT",
121
+ "dataset_name": "motion-example-video-clip-g1"
122
+ }
123
+ ]
124
+ JSON
125
+ )
126
+ fi
127
+
128
+ # 7-task from-scratch mix, excluding legacy VIDEO_TEXT / VIDEO_ONLY.
129
+ # Smaller data sources get larger weights. These use a tempered inverse-size
130
+ # rule, (47M / source_size)^0.25, so rare modalities are upweighted without
131
+ # completely starving text/video_clip in BatchMixtureSampler.
132
+ # text stride1 frame samples: ~47,000,000
133
+ # music audio dance frame samples: 213,731
134
+ # audio_command wav files: 41,760
135
+ # dense video_clip metadata: 4,597,576
136
+ PSI_TEXT_TASK_WEIGHT=1.0
137
+ PSI_AUDIO_TEXT_TASK_WEIGHT=3.8509
138
+ PSI_AUDIO_ONLY_TASK_WEIGHT=3.8509
139
+ PSI_AUDIO_COMMAND_TEXT_TASK_WEIGHT=5.7921
140
+ PSI_AUDIO_COMMAND_ONLY_TASK_WEIGHT=5.7921
141
+ PSI_VIDEO_TEXT_TASK_WEIGHT=0
142
+ PSI_VIDEO_ONLY_TASK_WEIGHT=0
143
+ PSI_VIDEO_CLIP_TEXT_TASK_WEIGHT=1.7881
144
+ PSI_VIDEO_CLIP_ONLY_TASK_WEIGHT=1.7881
145
+
146
+ PSI_PRETRAIN_BACKBONE_PATH=/cpfs01/VideoGen/public_ckpt/Qwen/Qwen2.5-Omni-3B
147
+ NUM_PAST_FRAMES=0
148
+ STATE_HORIZON=$((NUM_PAST_FRAMES + 1))
149
+ ACTION_CHUNK_SIZE=30
150
+ ACTION_EXEC_HORIZON=20
151
+ PROGRESS_DIM=5
152
+ VIDEO_SAMPLE_FPS=1.0
153
+ VIDEO_CLIP_SAMPLE_FPS=8.0
154
+ VIDEO_SIZE_PIXELS=50176
155
+ VIDEO_DECODER=ffmpeg
156
+ VIDEO_CACHE_SIZE=64
157
+ VIDEO_CLIP_MOTION_CACHE_SIZE=256
158
+ VIDEO_CLIP_TRUST_METADATA=${VIDEO_CLIP_TRUST_METADATA:-1}
159
+ VIDEO_CLIP_RECORDS_CACHE_DIR=${VIDEO_CLIP_RECORDS_CACHE_DIR:-$PSI_VIDEO_CLIP_DATASET_ROOT/.index_cache}
160
+ if [ "$VIDEO_CLIP_TRUST_METADATA" = "1" ] || [ "$VIDEO_CLIP_TRUST_METADATA" = "true" ] || [ "$VIDEO_CLIP_TRUST_METADATA" = "True" ]; then
161
+ VIDEO_CLIP_TRUST_METADATA_FLAG="--data.video-clip.trust-metadata"
162
+ else
163
+ VIDEO_CLIP_TRUST_METADATA_FLAG="--data.video-clip.no-trust-metadata"
164
+ fi
165
+ DATALOADER_NUM_WORKERS=4
166
+ VAL_DATALOADER_NUM_WORKERS=2
167
+
168
+ NPROC_PER_NODE=$(echo $CUDA_VISIBLE_DEVICES | tr ',' '\n' | wc -l)
169
+
170
+ echo "Training with $NPROC_PER_NODE GPU(s): $CUDA_VISIBLE_DEVICES"
171
+ echo "Text root: $PSI_SEED_ROOT"
172
+ echo "Dance motion root: $PSI_DANCE_MOTION_ROOT"
173
+ echo "Dance audio/video root: $PSI_DANCE_AUDIO_VIDEO_ROOT"
174
+ echo "Dance video root: $PSI_DANCE_VIDEO_ROOT"
175
+ echo "Dance video embed root: $PSI_DANCE_VIDEO_EMBED_ROOT"
176
+ echo "Qwen3-TTS acdudio command root: $PSI_QWEN3_TTS_ROOT"
177
+ echo "Audio sources: $PSI_AUDIO_SOURCES_JSON"
178
+ echo "Audio command sources: $PSI_AUDIO_COMMAND_SOURCES_JSON"
179
+ echo "Video sources: $PSI_VIDEO_SOURCES_JSON"
180
+ echo "Video clip sources: $PSI_VIDEO_CLIP_SOURCES_JSON"
181
+ echo "Task weights: text=$PSI_TEXT_TASK_WEIGHT audio_text=$PSI_AUDIO_TEXT_TASK_WEIGHT audio_only=$PSI_AUDIO_ONLY_TASK_WEIGHT audio_command_text=$PSI_AUDIO_COMMAND_TEXT_TASK_WEIGHT audio_command_only=$PSI_AUDIO_COMMAND_ONLY_TASK_WEIGHT video_text=$PSI_VIDEO_TEXT_TASK_WEIGHT video_only=$PSI_VIDEO_ONLY_TASK_WEIGHT video_clip_text=$PSI_VIDEO_CLIP_TEXT_TASK_WEIGHT video_clip_only=$PSI_VIDEO_CLIP_ONLY_TASK_WEIGHT"
182
+ # echo "Resume run dir: $PSI_PRETRAIN_RUN_DIR"
183
+ echo "TextOp 39D action/state enabled"
184
+ echo "Future progress embedding: enabled, dim $PROGRESS_DIM"
185
+ echo "Backbone path: $PSI_PRETRAIN_BACKBONE_PATH"
186
+ echo "Output dir: $PSI_OUTPUT_DIR"
187
+ echo "Video clip trust metadata: $VIDEO_CLIP_TRUST_METADATA"
188
+ echo "Video clip records cache dir: $VIDEO_CLIP_RECORDS_CACHE_DIR"
189
+
190
+ args="posttrain_v1_seed_text_audio_video_mix_psi0_config \
191
+ --seed=292285 \
192
+ --exp=posttrain \
193
+ --timestamp=$(date +"%y%m%d%H%M") \
194
+ --train.name=posttrain_v1_trimodal \
195
+ --train.data_parallel=ddp \
196
+ --train.mixed_precision=bf16 \
197
+ --train.train_batch_size=32 \
198
+ --train.val_batch_size=64 \
199
+ --train.num-workers=$DATALOADER_NUM_WORKERS \
200
+ --train.val-num-workers=$VAL_DATALOADER_NUM_WORKERS \
201
+ --train.resume_from_checkpoint=None \
202
+ --train.max_checkpoints_to_keep=1 \
203
+ --train.gradient_accumulation_steps=1 \
204
+ --train.learning_rate=1e-4 \
205
+ --train.max_training_steps=500000 \
206
+ --train.warmup_ratio=None \
207
+ --train.warmup_steps=1000 \
208
+ --train.checkpointing_steps=1000 \
209
+ --train.validation_steps=1000 \
210
+ --train.val_num_batches=20 \
211
+ --train.max_grad_norm=1.0 \
212
+ --train.lr_scheduler_type=cosine_with_min_lr \
213
+ --train.lr_scheduler_kwargs.betas 0.9 0.999 \
214
+ --train.lr_scheduler_kwargs.weight_decay=0 \
215
+ --train.lr_scheduler_kwargs.eps=1e-8 \
216
+ --log.report_to=$PSI_LOG_REPORT_TO \
217
+ --train.output_dir=$PSI_OUTPUT_DIR \
218
+ --data.text.root-dir=$PSI_SEED_ROOT \
219
+ --data.text.metadata-csv=$PSI_SEED_METADATA \
220
+ --data.text-ratio=$PSI_TEXT_TASK_WEIGHT \
221
+ --data.audio.sources-json=$PSI_AUDIO_SOURCES_JSON \
222
+ --data.audio-ratio=$PSI_AUDIO_TEXT_TASK_WEIGHT \
223
+ --data.audio-only-ratio=$PSI_AUDIO_ONLY_TASK_WEIGHT \
224
+ --data.audio.sample-stride=1 \
225
+ --data.audio.motion-fps=50 \
226
+ --data.audio.audio-sampling-rate=16000 \
227
+ --data.audio.audio-window-seconds=1.0 \
228
+ --data.audio.audio-left-context-seconds=0.0 \
229
+ --data.audio.full-audio \
230
+ --data.audio-command.sources-json=$PSI_AUDIO_COMMAND_SOURCES_JSON \
231
+ --data.audio-command-ratio=$PSI_AUDIO_COMMAND_TEXT_TASK_WEIGHT \
232
+ --data.audio-command-only-ratio=$PSI_AUDIO_COMMAND_ONLY_TASK_WEIGHT \
233
+ --data.audio-command.sample-stride=1 \
234
+ --data.audio-command.motion-fps=50 \
235
+ --data.audio-command.audio-sampling-rate=16000 \
236
+ --data.audio-command.audio-window-seconds=1.0 \
237
+ --data.audio-command.audio-left-context-seconds=0.0 \
238
+ --data.audio-command.full-audio \
239
+ --data.video-ratio=$PSI_VIDEO_TEXT_TASK_WEIGHT \
240
+ --data.video-only-ratio=$PSI_VIDEO_ONLY_TASK_WEIGHT \
241
+ --data.video.sample-stride=1 \
242
+ --data.video.motion-fps=50 \
243
+ --data.video.video-sample-fps=$VIDEO_SAMPLE_FPS \
244
+ --data.video.video-size-pixels=$VIDEO_SIZE_PIXELS \
245
+ --data.video.video-decoder=$VIDEO_DECODER \
246
+ --data.video.video-cache-size=$VIDEO_CACHE_SIZE \
247
+ --data.video.sources-json=$PSI_VIDEO_SOURCES_JSON \
248
+ --data.video-clip-text-ratio=$PSI_VIDEO_CLIP_TEXT_TASK_WEIGHT \
249
+ --data.video-clip-only-ratio=$PSI_VIDEO_CLIP_ONLY_TASK_WEIGHT \
250
+ --data.video-clip.sources-json=$PSI_VIDEO_CLIP_SOURCES_JSON \
251
+ --data.video-clip.sample-stride=1 \
252
+ --data.video-clip.motion-fps=50 \
253
+ --data.video-clip.video-sample-fps=$VIDEO_CLIP_SAMPLE_FPS \
254
+ --data.video-clip.video-size-pixels=$VIDEO_SIZE_PIXELS \
255
+ --data.video-clip.video-decoder=$VIDEO_DECODER \
256
+ --data.video-clip.video-cache-size=$VIDEO_CACHE_SIZE \
257
+ --data.video-clip.motion-cache-size=$VIDEO_CLIP_MOTION_CACHE_SIZE \
258
+ $VIDEO_CLIP_TRUST_METADATA_FLAG \
259
+ --data.video-clip.records-cache-dir=$VIDEO_CLIP_RECORDS_CACHE_DIR \
260
+ --data.video-clip.action-exec-horizon=$ACTION_EXEC_HORIZON \
261
+ --data.sampler=batch_mixture \
262
+ --data.val-task=text,audio,audio_only,audio_command,audio_command_only,video_clip,video_clip_only \
263
+ --data.transform.repack.num-past-frames=$NUM_PAST_FRAMES \
264
+ --data.transform.repack.textop-actions \
265
+ --data.transform.repack.action-chunk-size=$ACTION_CHUNK_SIZE \
266
+ --data.transform.repack.pad-action-dim=39 \
267
+ --data.transform.repack.pad-state-dim=39 \
268
+ --data.transform.field.action_norm_type=bounds_q99 \
269
+ --data.transform.field.stat-path=assets/stats/seed_all_g1_motion39_textop_stats.json \
270
+ --data.transform.field.normalize-state \
271
+ --data.transform.field.pad-action-dim=39 \
272
+ --data.transform.field.pad-state-dim=39 \
273
+ --data.transform.model.resize.size 240 320 \
274
+ --data.transform.model.center_crop.size 240 320 \
275
+ --data.transform.model.no-img-aug \
276
+ --data.transform.model.text-only \
277
+ --data.transform.model.audio-only \
278
+ --data.transform.model.video-only \
279
+ --data.transform.model.video-size-pixels=$VIDEO_SIZE_PIXELS \
280
+ --model.model_name_or_path=$PSI_PRETRAIN_BACKBONE_PATH \
281
+ --model.noise-scheduler=flow \
282
+ --model.n_conditions=0 \
283
+ --model.action-chunk-size=$ACTION_CHUNK_SIZE \
284
+ --model.action-dim=39 \
285
+ --model.action-exec-horizon=$ACTION_EXEC_HORIZON \
286
+ --model.observation-horizon=$STATE_HORIZON \
287
+ --model.odim=39 \
288
+ --model.view_feature_dim=2048 \
289
+ --model.use-future-progress-embedding \
290
+ --model.future-progress-dim=$PROGRESS_DIM \
291
+ --model.tune-vlm \
292
+ --model.lang-backbone-lr=1e-5 \
293
+ --model.no-use_film \
294
+ --model.no-combined_temb \
295
+ --model.rtc \
296
+ --model.max-delay=11 \
297
+ --train.max-grad-value=1 \
298
+ --train.loss-group-weight.joint-abs=1.0 \
299
+ --train.loss-group-weight.root-delta-xy=3.0 \
300
+ --train.loss-group-weight.root-rot5-delta=10.0 \
301
+ --train.loss-group-weight.root-height=15.0 \
302
+ --train.loss-group-weight.foot-contact=0.1
303
+ "
304
+ torchrun --nproc_per_node=$NPROC_PER_NODE --master_port=29500 scripts/train.py \
305
+ $args "$@"
posttrain-v1-psi0-all-text-tune-vlm.sh ADDED
@@ -0,0 +1,282 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ pkill -f monitor.py
2
+
3
+ cd /cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/Psi0-yth-v18
4
+ export PYTHONPATH=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/Psi0-yth-v18/src
5
+ export CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7
6
+ export PSI_OUTPUT_DIR=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/Psi0-yth-v18/runs
7
+ export PSI_LOG_REPORT_TO=wandb
8
+
9
+ source .venv-psi/bin/activate
10
+ [ -f .env ] && source .env
11
+
12
+ export WANDB_API_KEY="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"
13
+
14
+ export OMP_NUM_THREADS=8
15
+ export TF_CPP_MIN_LOG_LEVEL=3
16
+
17
+ # Keep JSON sources readable in this script, but pass compact JSON to tyro.
18
+ compact_json() {
19
+ python -c 'import json, sys; print(json.dumps(json.load(sys.stdin), separators=(",", ":")))'
20
+ }
21
+
22
+ PSI_SEED_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/Psi0/dataset/seed_all
23
+ PSI_SEED_METADATA=metadata/seed_metadata_v004.csv
24
+
25
+ PSI_DANCE_MOTION_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/GR00T-WholeBodyControl/gear_sonic_deploy/reference/converted/psi0_dance/gvhmr
26
+ PSI_DANCE_AUDIO_VIDEO_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/dance_1723_video_psi0/dance_1723_video_10s
27
+ PSI_DANCE_VIDEO_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/dance_1723_video_psi0/dance_1723_video_10s
28
+ PSI_DANCE_VIDEO_EMBED_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/dance_1723_video_psi0/dance_1723_video_10s_embeds
29
+ PSI_VIDEO_DEMO_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/videogen/motion_example_0506_demo
30
+ PSI_QWEN3_TTS_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/qwen3-tts
31
+ PSI_VIDEO_CLIP_DATASET_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/videogen/psi0_video_clip_50f_rand8_full
32
+ PSI_DANCE_VIDEO_CLIP_ROOT=$PSI_VIDEO_CLIP_DATASET_ROOT/dance_1723_video_10s
33
+ PSI_MOTION_EXAMPLE_VIDEO_CLIP_ROOT=$PSI_VIDEO_CLIP_DATASET_ROOT/motion_example_0506_demo
34
+ PSI_MOTION_EXAMPLE_VIDEO_EMBED_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/videogen/motion_example_0506_demo_video_embeds
35
+
36
+ if [ -z "$PSI_AUDIO_SOURCES_JSON" ]; then
37
+ PSI_AUDIO_SOURCES_JSON=$(compact_json <<JSON
38
+ [
39
+ {
40
+ "name": "psi0_dance_audio",
41
+ "root_dir": "$PSI_DANCE_MOTION_ROOT",
42
+ "video_dir": "$PSI_DANCE_AUDIO_VIDEO_ROOT",
43
+ "dataset_name": "seed-audio-g1"
44
+ }
45
+ ]
46
+ JSON
47
+ )
48
+ fi
49
+
50
+ if [ -z "$PSI_AUDIO_COMMAND_SOURCES_JSON" ]; then
51
+ PSI_AUDIO_COMMAND_SOURCES_JSON=$(compact_json <<JSON
52
+ [
53
+ {
54
+ "name": "qwen3_tts_seed_audio_command",
55
+ "root_dir": "$PSI_SEED_ROOT",
56
+ "dataset_root": "$PSI_QWEN3_TTS_ROOT",
57
+ "metadata_jsonl": "metadata.jsonl",
58
+ "motion_dir_key": "filename",
59
+ "audio_path_key": "audio_path",
60
+ "instruction_key": "content_short_description",
61
+ "motion_metadata_csv": "$PSI_SEED_METADATA",
62
+ "motion_metadata_key": "filename",
63
+ "motion_metadata_path_key": "move_g1_path",
64
+ "audio_file_extensions": ".wav",
65
+ "audio_task": "command",
66
+ "dataset_name": "seed-audio-command-g1"
67
+ }
68
+ ]
69
+ JSON
70
+ )
71
+ fi
72
+
73
+ if [ -z "$PSI_VIDEO_SOURCES_JSON" ]; then
74
+ PSI_VIDEO_SOURCES_JSON=$(compact_json <<JSON
75
+ [
76
+ {
77
+ "name": "psi0_dance_gvhmr",
78
+ "root_dir": "$PSI_DANCE_MOTION_ROOT",
79
+ "video_dir": "$PSI_DANCE_VIDEO_ROOT",
80
+ "video_embed_dir": "$PSI_DANCE_VIDEO_EMBED_ROOT",
81
+ "dataset_name": "seed-video-g1"
82
+ },
83
+ {
84
+ "name": "motion_example_0506_demo",
85
+ "dataset_root": "$PSI_VIDEO_DEMO_ROOT",
86
+ "metadata_jsonl": "metadata.jsonl",
87
+ "motion_dir_key": "converted_path",
88
+ "video_path_key": "video_path",
89
+ "instruction_key": "video_caption",
90
+ "video_embed_dir": "$PSI_MOTION_EXAMPLE_VIDEO_EMBED_ROOT",
91
+ "dataset_name": "motion-example-video-g1"
92
+ }
93
+ ]
94
+ JSON
95
+ )
96
+ fi
97
+
98
+ if [ -z "$PSI_VIDEO_CLIP_SOURCES_JSON" ]; then
99
+ PSI_VIDEO_CLIP_SOURCES_JSON=$(compact_json <<JSON
100
+ [
101
+ {
102
+ "name": "dance_1723_video_10s_clip50",
103
+ "dataset_root": "$PSI_DANCE_VIDEO_CLIP_ROOT",
104
+ "metadata_jsonl": "metadata.jsonl",
105
+ "motion_dir_key": "converted_path",
106
+ "video_path_key": "video_path",
107
+ "instruction_key": "video_caption",
108
+ "video_embed_dir": "$PSI_DANCE_VIDEO_CLIP_ROOT/embeds",
109
+ "dataset_name": "dance-video-clip-g1"
110
+ },
111
+ {
112
+ "name": "motion_example_0506_demo_clip50",
113
+ "dataset_root": "$PSI_MOTION_EXAMPLE_VIDEO_CLIP_ROOT",
114
+ "metadata_jsonl": "metadata.jsonl",
115
+ "motion_dir_key": "converted_path",
116
+ "video_path_key": "video_path",
117
+ "instruction_key": "video_caption",
118
+ "video_embed_dir": "$PSI_MOTION_EXAMPLE_VIDEO_CLIP_ROOT/embeds",
119
+ "dataset_name": "motion-example-video-clip-g1"
120
+ }
121
+ ]
122
+ JSON
123
+ )
124
+ fi
125
+
126
+ PSI_TEXT_TASK_WEIGHT=1
127
+ PSI_AUDIO_TEXT_TASK_WEIGHT=0
128
+ PSI_AUDIO_ONLY_TASK_WEIGHT=0
129
+ PSI_AUDIO_COMMAND_TEXT_TASK_WEIGHT=0
130
+ PSI_AUDIO_COMMAND_ONLY_TASK_WEIGHT=0
131
+ PSI_VIDEO_TEXT_TASK_WEIGHT=0
132
+ PSI_VIDEO_ONLY_TASK_WEIGHT=0
133
+ PSI_VIDEO_CLIP_TEXT_TASK_WEIGHT=0
134
+ PSI_VIDEO_CLIP_ONLY_TASK_WEIGHT=0
135
+
136
+ PSI_PRETRAIN_BACKBONE_PATH=/cpfs01/VideoGen/public_ckpt/Qwen/Qwen2.5-Omni-3B
137
+ NUM_PAST_FRAMES=0
138
+ STATE_HORIZON=$((NUM_PAST_FRAMES + 1))
139
+ ACTION_CHUNK_SIZE=30
140
+ ACTION_EXEC_HORIZON=20
141
+ PROGRESS_DIM=5
142
+ VIDEO_SAMPLE_FPS=1.0
143
+ VIDEO_CLIP_SAMPLE_FPS=8.0
144
+ VIDEO_SIZE_PIXELS=50176
145
+ VIDEO_DECODER=ffmpeg
146
+ VIDEO_CACHE_SIZE=64
147
+ DATALOADER_NUM_WORKERS=4
148
+ VAL_DATALOADER_NUM_WORKERS=2
149
+
150
+ NPROC_PER_NODE=$(echo $CUDA_VISIBLE_DEVICES | tr ',' '\n' | wc -l)
151
+
152
+ echo "Training with $NPROC_PER_NODE GPU(s): $CUDA_VISIBLE_DEVICES"
153
+ echo "Text root: $PSI_SEED_ROOT"
154
+ echo "Dance motion root: $PSI_DANCE_MOTION_ROOT"
155
+ echo "Dance audio/video root: $PSI_DANCE_AUDIO_VIDEO_ROOT"
156
+ echo "Dance video root: $PSI_DANCE_VIDEO_ROOT"
157
+ echo "Dance video embed root: $PSI_DANCE_VIDEO_EMBED_ROOT"
158
+ echo "Qwen3-TTS acdudio command root: $PSI_QWEN3_TTS_ROOT"
159
+ echo "Audio sources: $PSI_AUDIO_SOURCES_JSON"
160
+ echo "Audio command sources: $PSI_AUDIO_COMMAND_SOURCES_JSON"
161
+ echo "Video sources: $PSI_VIDEO_SOURCES_JSON"
162
+ echo "Video clip sources: $PSI_VIDEO_CLIP_SOURCES_JSON"
163
+ echo "Task weights: text=$PSI_TEXT_TASK_WEIGHT audio_text=$PSI_AUDIO_TEXT_TASK_WEIGHT audio_only=$PSI_AUDIO_ONLY_TASK_WEIGHT audio_command_text=$PSI_AUDIO_COMMAND_TEXT_TASK_WEIGHT audio_command_only=$PSI_AUDIO_COMMAND_ONLY_TASK_WEIGHT video_text=$PSI_VIDEO_TEXT_TASK_WEIGHT video_only=$PSI_VIDEO_ONLY_TASK_WEIGHT video_clip_text=$PSI_VIDEO_CLIP_TEXT_TASK_WEIGHT video_clip_only=$PSI_VIDEO_CLIP_ONLY_TASK_WEIGHT"
164
+ # echo "Resume run dir: $PSI_PRETRAIN_RUN_DIR"
165
+ echo "TextOp 39D action/state enabled"
166
+ echo "Future progress embedding: enabled, dim $PROGRESS_DIM"
167
+ echo "Backbone path: $PSI_PRETRAIN_BACKBONE_PATH"
168
+ echo "Output dir: $PSI_OUTPUT_DIR"
169
+
170
+ args="posttrain_v1_seed_text_audio_video_mix_psi0_config \
171
+ --seed=292285 \
172
+ --exp=posttrain \
173
+ --timestamp=$(date +"%y%m%d%H%M") \
174
+ --train.name=posttrain_v1_trimodal \
175
+ --train.data_parallel=ddp \
176
+ --train.mixed_precision=bf16 \
177
+ --train.train_batch_size=32 \
178
+ --train.val_batch_size=64 \
179
+ --train.num-workers=$DATALOADER_NUM_WORKERS \
180
+ --train.val-num-workers=$VAL_DATALOADER_NUM_WORKERS \
181
+ --train.resume_from_checkpoint=None \
182
+ --train.max_checkpoints_to_keep=1 \
183
+ --train.gradient_accumulation_steps=1 \
184
+ --train.learning_rate=1e-4 \
185
+ --train.max_training_steps=500000 \
186
+ --train.warmup_ratio=None \
187
+ --train.warmup_steps=1000 \
188
+ --train.checkpointing_steps=1000 \
189
+ --train.validation_steps=1000 \
190
+ --train.val_num_batches=20 \
191
+ --train.max_grad_norm=1.0 \
192
+ --train.lr_scheduler_type=cosine_with_min_lr \
193
+ --train.lr_scheduler_kwargs.betas 0.9 0.999 \
194
+ --train.lr_scheduler_kwargs.weight_decay=0 \
195
+ --train.lr_scheduler_kwargs.eps=1e-8 \
196
+ --log.report_to=$PSI_LOG_REPORT_TO \
197
+ --train.output_dir=$PSI_OUTPUT_DIR \
198
+ --data.text.root-dir=$PSI_SEED_ROOT \
199
+ --data.text.metadata-csv=$PSI_SEED_METADATA \
200
+ --data.text-ratio=$PSI_TEXT_TASK_WEIGHT \
201
+ --data.audio.sources-json=$PSI_AUDIO_SOURCES_JSON \
202
+ --data.audio-ratio=$PSI_AUDIO_TEXT_TASK_WEIGHT \
203
+ --data.audio-only-ratio=$PSI_AUDIO_ONLY_TASK_WEIGHT \
204
+ --data.audio.sample-stride=1 \
205
+ --data.audio.motion-fps=50 \
206
+ --data.audio.audio-sampling-rate=16000 \
207
+ --data.audio.audio-window-seconds=1.0 \
208
+ --data.audio.audio-left-context-seconds=0.0 \
209
+ --data.audio.full-audio \
210
+ --data.audio-command.sources-json=$PSI_AUDIO_COMMAND_SOURCES_JSON \
211
+ --data.audio-command-ratio=$PSI_AUDIO_COMMAND_TEXT_TASK_WEIGHT \
212
+ --data.audio-command-only-ratio=$PSI_AUDIO_COMMAND_ONLY_TASK_WEIGHT \
213
+ --data.audio-command.sample-stride=1 \
214
+ --data.audio-command.motion-fps=50 \
215
+ --data.audio-command.audio-sampling-rate=16000 \
216
+ --data.audio-command.audio-window-seconds=1.0 \
217
+ --data.audio-command.audio-left-context-seconds=0.0 \
218
+ --data.audio-command.full-audio \
219
+ --data.video-ratio=$PSI_VIDEO_TEXT_TASK_WEIGHT \
220
+ --data.video-only-ratio=$PSI_VIDEO_ONLY_TASK_WEIGHT \
221
+ --data.video.sample-stride=1 \
222
+ --data.video.motion-fps=50 \
223
+ --data.video.video-sample-fps=$VIDEO_SAMPLE_FPS \
224
+ --data.video.video-size-pixels=$VIDEO_SIZE_PIXELS \
225
+ --data.video.video-decoder=$VIDEO_DECODER \
226
+ --data.video.video-cache-size=$VIDEO_CACHE_SIZE \
227
+ --data.video.sources-json=$PSI_VIDEO_SOURCES_JSON \
228
+ --data.video-clip-text-ratio=$PSI_VIDEO_CLIP_TEXT_TASK_WEIGHT \
229
+ --data.video-clip-only-ratio=$PSI_VIDEO_CLIP_ONLY_TASK_WEIGHT \
230
+ --data.video-clip.sources-json=$PSI_VIDEO_CLIP_SOURCES_JSON \
231
+ --data.video-clip.sample-stride=1 \
232
+ --data.video-clip.motion-fps=50 \
233
+ --data.video-clip.video-sample-fps=$VIDEO_CLIP_SAMPLE_FPS \
234
+ --data.video-clip.video-size-pixels=$VIDEO_SIZE_PIXELS \
235
+ --data.video-clip.video-decoder=$VIDEO_DECODER \
236
+ --data.video-clip.video-cache-size=$VIDEO_CACHE_SIZE \
237
+ --data.video-clip.action-exec-horizon=$ACTION_EXEC_HORIZON \
238
+ --data.sampler=batch_mixture \
239
+ --data.val-task=text \
240
+ --data.transform.repack.num-past-frames=$NUM_PAST_FRAMES \
241
+ --data.transform.repack.textop-actions \
242
+ --data.transform.repack.action-chunk-size=$ACTION_CHUNK_SIZE \
243
+ --data.transform.repack.pad-action-dim=39 \
244
+ --data.transform.repack.pad-state-dim=39 \
245
+ --data.transform.field.action_norm_type=bounds_q99 \
246
+ --data.transform.field.stat-path=assets/stats/seed_all_g1_motion39_textop_stats.json \
247
+ --data.transform.field.normalize-state \
248
+ --data.transform.field.pad-action-dim=39 \
249
+ --data.transform.field.pad-state-dim=39 \
250
+ --data.transform.model.resize.size 240 320 \
251
+ --data.transform.model.center_crop.size 240 320 \
252
+ --data.transform.model.no-img-aug \
253
+ --data.transform.model.text-only \
254
+ --data.transform.model.audio-only \
255
+ --data.transform.model.video-only \
256
+ --data.transform.model.video-size-pixels=$VIDEO_SIZE_PIXELS \
257
+ --model.model_name_or_path=$PSI_PRETRAIN_BACKBONE_PATH \
258
+ --model.noise-scheduler=flow \
259
+ --model.n_conditions=0 \
260
+ --model.action-chunk-size=$ACTION_CHUNK_SIZE \
261
+ --model.action-dim=39 \
262
+ --model.action-exec-horizon=$ACTION_EXEC_HORIZON \
263
+ --model.observation-horizon=$STATE_HORIZON \
264
+ --model.odim=39 \
265
+ --model.view_feature_dim=2048 \
266
+ --model.use-future-progress-embedding \
267
+ --model.future-progress-dim=$PROGRESS_DIM \
268
+ --model.tune-vlm \
269
+ --model.lang-backbone-lr=1e-5 \
270
+ --model.no-use_film \
271
+ --model.no-combined_temb \
272
+ --model.rtc \
273
+ --model.max-delay=11 \
274
+ --train.max-grad-value=1 \
275
+ --train.loss-group-weight.joint-abs=1.0 \
276
+ --train.loss-group-weight.root-delta-xy=3.0 \
277
+ --train.loss-group-weight.root-rot5-delta=10.0 \
278
+ --train.loss-group-weight.root-height=15.0 \
279
+ --train.loss-group-weight.foot-contact=0.1
280
+ "
281
+ torchrun --nproc_per_node=$NPROC_PER_NODE --master_port=29500 scripts/train.py \
282
+ $args "$@"
posttrain-v1-psi0-all-tune-vlm-direct-dense.sh ADDED
@@ -0,0 +1,297 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ pkill -f monitor.py
2
+
3
+ cd /cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/Psi0-yth-v18
4
+ export PYTHONPATH=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/Psi0-yth-v18/src
5
+ export CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7
6
+ export PSI_OUTPUT_DIR=${PSI_OUTPUT_DIR:-/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/Psi0-yth-v18/runs}
7
+ export PSI_LOG_REPORT_TO=wandb
8
+
9
+ source /cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/Psi0-yth-v15/.venv-psi/bin/activate
10
+ [ -f .env ] && source .env
11
+
12
+ export WANDB_API_KEY="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"
13
+
14
+ export OMP_NUM_THREADS=8
15
+ export TF_CPP_MIN_LOG_LEVEL=3
16
+
17
+ # Keep JSON sources readable in this script, but pass compact JSON to tyro.
18
+ compact_json() {
19
+ python -c 'import json, sys; print(json.dumps(json.load(sys.stdin), separators=(",", ":")))'
20
+ }
21
+
22
+ PSI_SEED_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/Psi0/dataset/seed_all
23
+ PSI_SEED_METADATA=metadata/seed_metadata_v004.csv
24
+
25
+ PSI_DANCE_MOTION_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/GR00T-WholeBodyControl/gear_sonic_deploy/reference/converted/psi0_dance/gvhmr
26
+ PSI_DANCE_AUDIO_VIDEO_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/dance_1723_video_psi0/dance_1723_video_10s
27
+ PSI_DANCE_VIDEO_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/dance_1723_video_psi0/dance_1723_video_10s
28
+ PSI_DANCE_VIDEO_EMBED_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/dance_1723_video_psi0/dance_1723_video_10s_embeds
29
+ PSI_VIDEO_DEMO_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/videogen/motion_example_0506_demo
30
+ PSI_QWEN3_TTS_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/qwen3-tts
31
+ PSI_VIDEO_CLIP_DATASET_ROOT=${PSI_VIDEO_CLIP_DATASET_ROOT:-/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/videogen/psi0_video_clip_50f_dense_stride1_direct_full}
32
+ PSI_DANCE_VIDEO_CLIP_ROOT=$PSI_VIDEO_CLIP_DATASET_ROOT/dance_1723_video_10s
33
+ PSI_MOTION_EXAMPLE_VIDEO_CLIP_ROOT=$PSI_VIDEO_CLIP_DATASET_ROOT/motion_example_0506_demo
34
+ PSI_DANCE_VIDEO_CLIP_EMBED_ROOT=${PSI_DANCE_VIDEO_CLIP_EMBED_ROOT:-$PSI_DANCE_VIDEO_CLIP_ROOT/embeds}
35
+ PSI_MOTION_EXAMPLE_VIDEO_CLIP_EMBED_ROOT=${PSI_MOTION_EXAMPLE_VIDEO_CLIP_EMBED_ROOT:-$PSI_MOTION_EXAMPLE_VIDEO_CLIP_ROOT/embeds}
36
+ PSI_MOTION_EXAMPLE_VIDEO_EMBED_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/videogen/motion_example_0506_demo_video_embeds
37
+
38
+ if [ -z "$PSI_AUDIO_SOURCES_JSON" ]; then
39
+ PSI_AUDIO_SOURCES_JSON=$(compact_json <<JSON
40
+ [
41
+ {
42
+ "name": "psi0_dance_audio",
43
+ "root_dir": "$PSI_DANCE_MOTION_ROOT",
44
+ "video_dir": "$PSI_DANCE_AUDIO_VIDEO_ROOT",
45
+ "dataset_name": "seed-audio-g1"
46
+ }
47
+ ]
48
+ JSON
49
+ )
50
+ fi
51
+
52
+ if [ -z "$PSI_AUDIO_COMMAND_SOURCES_JSON" ]; then
53
+ PSI_AUDIO_COMMAND_SOURCES_JSON=$(compact_json <<JSON
54
+ [
55
+ {
56
+ "name": "qwen3_tts_seed_audio_command",
57
+ "root_dir": "$PSI_SEED_ROOT",
58
+ "dataset_root": "$PSI_QWEN3_TTS_ROOT",
59
+ "metadata_jsonl": "metadata.jsonl",
60
+ "motion_dir_key": "filename",
61
+ "audio_path_key": "audio_path",
62
+ "instruction_key": "content_short_description",
63
+ "motion_metadata_csv": "$PSI_SEED_METADATA",
64
+ "motion_metadata_key": "filename",
65
+ "motion_metadata_path_key": "move_g1_path",
66
+ "audio_file_extensions": ".wav",
67
+ "audio_task": "command",
68
+ "dataset_name": "seed-audio-command-g1"
69
+ }
70
+ ]
71
+ JSON
72
+ )
73
+ fi
74
+
75
+ if [ -z "$PSI_VIDEO_SOURCES_JSON" ]; then
76
+ PSI_VIDEO_SOURCES_JSON=$(compact_json <<JSON
77
+ [
78
+ {
79
+ "name": "psi0_dance_gvhmr",
80
+ "root_dir": "$PSI_DANCE_MOTION_ROOT",
81
+ "video_dir": "$PSI_DANCE_VIDEO_ROOT",
82
+ "video_embed_dir": "$PSI_DANCE_VIDEO_EMBED_ROOT",
83
+ "dataset_name": "seed-video-g1"
84
+ },
85
+ {
86
+ "name": "motion_example_0506_demo",
87
+ "dataset_root": "$PSI_VIDEO_DEMO_ROOT",
88
+ "metadata_jsonl": "metadata.jsonl",
89
+ "motion_dir_key": "converted_path",
90
+ "video_path_key": "video_path",
91
+ "instruction_key": "video_caption",
92
+ "video_embed_dir": "$PSI_MOTION_EXAMPLE_VIDEO_EMBED_ROOT",
93
+ "dataset_name": "motion-example-video-g1"
94
+ }
95
+ ]
96
+ JSON
97
+ )
98
+ fi
99
+
100
+ if [ -z "$PSI_VIDEO_CLIP_SOURCES_JSON" ]; then
101
+ PSI_VIDEO_CLIP_SOURCES_JSON=$(compact_json <<JSON
102
+ [
103
+ {
104
+ "name": "dance_1723_video_10s_clip50",
105
+ "dataset_root": "$PSI_DANCE_VIDEO_CLIP_ROOT",
106
+ "metadata_jsonl": "metadata.jsonl",
107
+ "motion_dir_key": "converted_path",
108
+ "video_path_key": "video_path",
109
+ "instruction_key": "video_caption",
110
+ "video_embed_dir": "$PSI_DANCE_VIDEO_CLIP_EMBED_ROOT",
111
+ "dataset_name": "dance-video-clip-g1"
112
+ },
113
+ {
114
+ "name": "motion_example_0506_demo_clip50",
115
+ "dataset_root": "$PSI_MOTION_EXAMPLE_VIDEO_CLIP_ROOT",
116
+ "metadata_jsonl": "metadata.jsonl",
117
+ "motion_dir_key": "converted_path",
118
+ "video_path_key": "video_path",
119
+ "instruction_key": "video_caption",
120
+ "video_embed_dir": "$PSI_MOTION_EXAMPLE_VIDEO_CLIP_EMBED_ROOT",
121
+ "dataset_name": "motion-example-video-clip-g1"
122
+ }
123
+ ]
124
+ JSON
125
+ )
126
+ fi
127
+
128
+ PSI_TEXT_TASK_WEIGHT=0
129
+ PSI_AUDIO_TEXT_TASK_WEIGHT=0
130
+ PSI_AUDIO_ONLY_TASK_WEIGHT=0
131
+ PSI_AUDIO_COMMAND_TEXT_TASK_WEIGHT=0
132
+ PSI_AUDIO_COMMAND_ONLY_TASK_WEIGHT=0
133
+ PSI_VIDEO_TEXT_TASK_WEIGHT=0
134
+ PSI_VIDEO_ONLY_TASK_WEIGHT=0
135
+ PSI_VIDEO_CLIP_TEXT_TASK_WEIGHT=0.5
136
+ PSI_VIDEO_CLIP_ONLY_TASK_WEIGHT=0.5
137
+
138
+ PSI_PRETRAIN_BACKBONE_PATH=/cpfs01/VideoGen/public_ckpt/Qwen/Qwen2.5-Omni-3B
139
+ NUM_PAST_FRAMES=0
140
+ STATE_HORIZON=$((NUM_PAST_FRAMES + 1))
141
+ ACTION_CHUNK_SIZE=30
142
+ ACTION_EXEC_HORIZON=20
143
+ PROGRESS_DIM=5
144
+ VIDEO_SAMPLE_FPS=1.0
145
+ VIDEO_CLIP_SAMPLE_FPS=8.0
146
+ VIDEO_SIZE_PIXELS=50176
147
+ VIDEO_DECODER=ffmpeg
148
+ VIDEO_CACHE_SIZE=64
149
+ VIDEO_CLIP_MOTION_CACHE_SIZE=256
150
+ VIDEO_CLIP_TRUST_METADATA=${VIDEO_CLIP_TRUST_METADATA:-1}
151
+ VIDEO_CLIP_RECORDS_CACHE_DIR=${VIDEO_CLIP_RECORDS_CACHE_DIR:-$PSI_VIDEO_CLIP_DATASET_ROOT/.index_cache}
152
+ if [ "$VIDEO_CLIP_TRUST_METADATA" = "1" ] || [ "$VIDEO_CLIP_TRUST_METADATA" = "true" ] || [ "$VIDEO_CLIP_TRUST_METADATA" = "True" ]; then
153
+ VIDEO_CLIP_TRUST_METADATA_FLAG="--data.video-clip.trust-metadata"
154
+ else
155
+ VIDEO_CLIP_TRUST_METADATA_FLAG="--data.video-clip.no-trust-metadata"
156
+ fi
157
+ DATALOADER_NUM_WORKERS=4
158
+ VAL_DATALOADER_NUM_WORKERS=2
159
+
160
+ NPROC_PER_NODE=$(echo $CUDA_VISIBLE_DEVICES | tr ',' '\n' | wc -l)
161
+
162
+ echo "Training with $NPROC_PER_NODE GPU(s): $CUDA_VISIBLE_DEVICES"
163
+ echo "Text root: $PSI_SEED_ROOT"
164
+ echo "Dance motion root: $PSI_DANCE_MOTION_ROOT"
165
+ echo "Dance audio/video root: $PSI_DANCE_AUDIO_VIDEO_ROOT"
166
+ echo "Dance video root: $PSI_DANCE_VIDEO_ROOT"
167
+ echo "Dance video embed root: $PSI_DANCE_VIDEO_EMBED_ROOT"
168
+ echo "Qwen3-TTS acdudio command root: $PSI_QWEN3_TTS_ROOT"
169
+ echo "Audio sources: $PSI_AUDIO_SOURCES_JSON"
170
+ echo "Audio command sources: $PSI_AUDIO_COMMAND_SOURCES_JSON"
171
+ echo "Video sources: $PSI_VIDEO_SOURCES_JSON"
172
+ echo "Video clip sources: $PSI_VIDEO_CLIP_SOURCES_JSON"
173
+ echo "Task weights: text=$PSI_TEXT_TASK_WEIGHT audio_text=$PSI_AUDIO_TEXT_TASK_WEIGHT audio_only=$PSI_AUDIO_ONLY_TASK_WEIGHT audio_command_text=$PSI_AUDIO_COMMAND_TEXT_TASK_WEIGHT audio_command_only=$PSI_AUDIO_COMMAND_ONLY_TASK_WEIGHT video_text=$PSI_VIDEO_TEXT_TASK_WEIGHT video_only=$PSI_VIDEO_ONLY_TASK_WEIGHT video_clip_text=$PSI_VIDEO_CLIP_TEXT_TASK_WEIGHT video_clip_only=$PSI_VIDEO_CLIP_ONLY_TASK_WEIGHT"
174
+ # echo "Resume run dir: $PSI_PRETRAIN_RUN_DIR"
175
+ echo "TextOp 39D action/state enabled"
176
+ echo "Future progress embedding: enabled, dim $PROGRESS_DIM"
177
+ echo "Backbone path: $PSI_PRETRAIN_BACKBONE_PATH"
178
+ echo "Output dir: $PSI_OUTPUT_DIR"
179
+ echo "Video clip trust metadata: $VIDEO_CLIP_TRUST_METADATA"
180
+ echo "Video clip records cache dir: $VIDEO_CLIP_RECORDS_CACHE_DIR"
181
+
182
+ args="posttrain_v1_seed_text_audio_video_mix_psi0_config \
183
+ --seed=292285 \
184
+ --exp=posttrain \
185
+ --timestamp=$(date +"%y%m%d%H%M") \
186
+ --train.name=posttrain_v1_trimodal \
187
+ --train.data_parallel=ddp \
188
+ --train.mixed_precision=bf16 \
189
+ --train.train_batch_size=32 \
190
+ --train.val_batch_size=64 \
191
+ --train.num-workers=$DATALOADER_NUM_WORKERS \
192
+ --train.val-num-workers=$VAL_DATALOADER_NUM_WORKERS \
193
+ --train.resume_from_checkpoint=None \
194
+ --train.max_checkpoints_to_keep=1 \
195
+ --train.gradient_accumulation_steps=1 \
196
+ --train.learning_rate=1e-4 \
197
+ --train.max_training_steps=500000 \
198
+ --train.warmup_ratio=None \
199
+ --train.warmup_steps=1000 \
200
+ --train.checkpointing_steps=1000 \
201
+ --train.validation_steps=1000 \
202
+ --train.val_num_batches=20 \
203
+ --train.max_grad_norm=1.0 \
204
+ --train.lr_scheduler_type=cosine_with_min_lr \
205
+ --train.lr_scheduler_kwargs.betas 0.9 0.999 \
206
+ --train.lr_scheduler_kwargs.weight_decay=0 \
207
+ --train.lr_scheduler_kwargs.eps=1e-8 \
208
+ --log.report_to=$PSI_LOG_REPORT_TO \
209
+ --train.output_dir=$PSI_OUTPUT_DIR \
210
+ --data.text.root-dir=$PSI_SEED_ROOT \
211
+ --data.text.metadata-csv=$PSI_SEED_METADATA \
212
+ --data.text-ratio=$PSI_TEXT_TASK_WEIGHT \
213
+ --data.audio.sources-json=$PSI_AUDIO_SOURCES_JSON \
214
+ --data.audio-ratio=$PSI_AUDIO_TEXT_TASK_WEIGHT \
215
+ --data.audio-only-ratio=$PSI_AUDIO_ONLY_TASK_WEIGHT \
216
+ --data.audio.sample-stride=1 \
217
+ --data.audio.motion-fps=50 \
218
+ --data.audio.audio-sampling-rate=16000 \
219
+ --data.audio.audio-window-seconds=1.0 \
220
+ --data.audio.audio-left-context-seconds=0.0 \
221
+ --data.audio.full-audio \
222
+ --data.audio-command.sources-json=$PSI_AUDIO_COMMAND_SOURCES_JSON \
223
+ --data.audio-command-ratio=$PSI_AUDIO_COMMAND_TEXT_TASK_WEIGHT \
224
+ --data.audio-command-only-ratio=$PSI_AUDIO_COMMAND_ONLY_TASK_WEIGHT \
225
+ --data.audio-command.sample-stride=1 \
226
+ --data.audio-command.motion-fps=50 \
227
+ --data.audio-command.audio-sampling-rate=16000 \
228
+ --data.audio-command.audio-window-seconds=1.0 \
229
+ --data.audio-command.audio-left-context-seconds=0.0 \
230
+ --data.audio-command.full-audio \
231
+ --data.video-ratio=$PSI_VIDEO_TEXT_TASK_WEIGHT \
232
+ --data.video-only-ratio=$PSI_VIDEO_ONLY_TASK_WEIGHT \
233
+ --data.video.sample-stride=1 \
234
+ --data.video.motion-fps=50 \
235
+ --data.video.video-sample-fps=$VIDEO_SAMPLE_FPS \
236
+ --data.video.video-size-pixels=$VIDEO_SIZE_PIXELS \
237
+ --data.video.video-decoder=$VIDEO_DECODER \
238
+ --data.video.video-cache-size=$VIDEO_CACHE_SIZE \
239
+ --data.video.sources-json=$PSI_VIDEO_SOURCES_JSON \
240
+ --data.video-clip-text-ratio=$PSI_VIDEO_CLIP_TEXT_TASK_WEIGHT \
241
+ --data.video-clip-only-ratio=$PSI_VIDEO_CLIP_ONLY_TASK_WEIGHT \
242
+ --data.video-clip.sources-json=$PSI_VIDEO_CLIP_SOURCES_JSON \
243
+ --data.video-clip.sample-stride=1 \
244
+ --data.video-clip.motion-fps=50 \
245
+ --data.video-clip.video-sample-fps=$VIDEO_CLIP_SAMPLE_FPS \
246
+ --data.video-clip.video-size-pixels=$VIDEO_SIZE_PIXELS \
247
+ --data.video-clip.video-decoder=$VIDEO_DECODER \
248
+ --data.video-clip.video-cache-size=$VIDEO_CACHE_SIZE \
249
+ --data.video-clip.motion-cache-size=$VIDEO_CLIP_MOTION_CACHE_SIZE \
250
+ $VIDEO_CLIP_TRUST_METADATA_FLAG \
251
+ --data.video-clip.records-cache-dir=$VIDEO_CLIP_RECORDS_CACHE_DIR \
252
+ --data.video-clip.action-exec-horizon=$ACTION_EXEC_HORIZON \
253
+ --data.sampler=batch_mixture \
254
+ --data.val-task=video_clip,video_clip_only \
255
+ --data.transform.repack.num-past-frames=$NUM_PAST_FRAMES \
256
+ --data.transform.repack.textop-actions \
257
+ --data.transform.repack.action-chunk-size=$ACTION_CHUNK_SIZE \
258
+ --data.transform.repack.pad-action-dim=39 \
259
+ --data.transform.repack.pad-state-dim=39 \
260
+ --data.transform.field.action_norm_type=bounds_q99 \
261
+ --data.transform.field.stat-path=assets/stats/seed_all_g1_motion39_textop_stats.json \
262
+ --data.transform.field.normalize-state \
263
+ --data.transform.field.pad-action-dim=39 \
264
+ --data.transform.field.pad-state-dim=39 \
265
+ --data.transform.model.resize.size 240 320 \
266
+ --data.transform.model.center_crop.size 240 320 \
267
+ --data.transform.model.no-img-aug \
268
+ --data.transform.model.text-only \
269
+ --data.transform.model.audio-only \
270
+ --data.transform.model.video-only \
271
+ --data.transform.model.video-size-pixels=$VIDEO_SIZE_PIXELS \
272
+ --model.model_name_or_path=$PSI_PRETRAIN_BACKBONE_PATH \
273
+ --model.noise-scheduler=flow \
274
+ --model.n_conditions=0 \
275
+ --model.action-chunk-size=$ACTION_CHUNK_SIZE \
276
+ --model.action-dim=39 \
277
+ --model.action-exec-horizon=$ACTION_EXEC_HORIZON \
278
+ --model.observation-horizon=$STATE_HORIZON \
279
+ --model.odim=39 \
280
+ --model.view_feature_dim=2048 \
281
+ --model.use-future-progress-embedding \
282
+ --model.future-progress-dim=$PROGRESS_DIM \
283
+ --model.tune-vlm \
284
+ --model.lang-backbone-lr=1e-5 \
285
+ --model.no-use_film \
286
+ --model.no-combined_temb \
287
+ --model.rtc \
288
+ --model.max-delay=11 \
289
+ --train.max-grad-value=1 \
290
+ --train.loss-group-weight.joint-abs=1.0 \
291
+ --train.loss-group-weight.root-delta-xy=3.0 \
292
+ --train.loss-group-weight.root-rot5-delta=10.0 \
293
+ --train.loss-group-weight.root-height=15.0 \
294
+ --train.loss-group-weight.foot-contact=0.1
295
+ "
296
+ torchrun --nproc_per_node=$NPROC_PER_NODE --master_port=29500 scripts/train.py \
297
+ $args "$@"
posttrain-v1-psi0-all-tune-vlm.sh ADDED
@@ -0,0 +1,282 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ pkill -f monitor.py
2
+
3
+ cd /cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/Psi0-yth-v18
4
+ export PYTHONPATH=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/Psi0-yth-v18/src
5
+ export CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7
6
+ export PSI_OUTPUT_DIR=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/Psi0-yth-v18/runs
7
+ export PSI_LOG_REPORT_TO=wandb
8
+
9
+ source .venv-psi/bin/activate
10
+ [ -f .env ] && source .env
11
+
12
+ export WANDB_API_KEY="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"
13
+
14
+ export OMP_NUM_THREADS=8
15
+ export TF_CPP_MIN_LOG_LEVEL=3
16
+
17
+ # Keep JSON sources readable in this script, but pass compact JSON to tyro.
18
+ compact_json() {
19
+ python -c 'import json, sys; print(json.dumps(json.load(sys.stdin), separators=(",", ":")))'
20
+ }
21
+
22
+ PSI_SEED_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/Psi0/dataset/seed_all
23
+ PSI_SEED_METADATA=metadata/seed_metadata_v004.csv
24
+
25
+ PSI_DANCE_MOTION_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/GR00T-WholeBodyControl/gear_sonic_deploy/reference/converted/psi0_dance/gvhmr
26
+ PSI_DANCE_AUDIO_VIDEO_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/dance_1723_video_psi0/dance_1723_video_10s
27
+ PSI_DANCE_VIDEO_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/dance_1723_video_psi0/dance_1723_video_10s
28
+ PSI_DANCE_VIDEO_EMBED_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/dance_1723_video_psi0/dance_1723_video_10s_embeds
29
+ PSI_VIDEO_DEMO_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/videogen/motion_example_0506_demo
30
+ PSI_QWEN3_TTS_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/qwen3-tts
31
+ PSI_VIDEO_CLIP_DATASET_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/videogen/psi0_video_clip_50f_rand8_full
32
+ PSI_DANCE_VIDEO_CLIP_ROOT=$PSI_VIDEO_CLIP_DATASET_ROOT/dance_1723_video_10s
33
+ PSI_MOTION_EXAMPLE_VIDEO_CLIP_ROOT=$PSI_VIDEO_CLIP_DATASET_ROOT/motion_example_0506_demo
34
+ PSI_MOTION_EXAMPLE_VIDEO_EMBED_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/videogen/motion_example_0506_demo_video_embeds
35
+
36
+ if [ -z "$PSI_AUDIO_SOURCES_JSON" ]; then
37
+ PSI_AUDIO_SOURCES_JSON=$(compact_json <<JSON
38
+ [
39
+ {
40
+ "name": "psi0_dance_audio",
41
+ "root_dir": "$PSI_DANCE_MOTION_ROOT",
42
+ "video_dir": "$PSI_DANCE_AUDIO_VIDEO_ROOT",
43
+ "dataset_name": "seed-audio-g1"
44
+ }
45
+ ]
46
+ JSON
47
+ )
48
+ fi
49
+
50
+ if [ -z "$PSI_AUDIO_COMMAND_SOURCES_JSON" ]; then
51
+ PSI_AUDIO_COMMAND_SOURCES_JSON=$(compact_json <<JSON
52
+ [
53
+ {
54
+ "name": "qwen3_tts_seed_audio_command",
55
+ "root_dir": "$PSI_SEED_ROOT",
56
+ "dataset_root": "$PSI_QWEN3_TTS_ROOT",
57
+ "metadata_jsonl": "metadata.jsonl",
58
+ "motion_dir_key": "filename",
59
+ "audio_path_key": "audio_path",
60
+ "instruction_key": "content_short_description",
61
+ "motion_metadata_csv": "$PSI_SEED_METADATA",
62
+ "motion_metadata_key": "filename",
63
+ "motion_metadata_path_key": "move_g1_path",
64
+ "audio_file_extensions": ".wav",
65
+ "audio_task": "command",
66
+ "dataset_name": "seed-audio-command-g1"
67
+ }
68
+ ]
69
+ JSON
70
+ )
71
+ fi
72
+
73
+ if [ -z "$PSI_VIDEO_SOURCES_JSON" ]; then
74
+ PSI_VIDEO_SOURCES_JSON=$(compact_json <<JSON
75
+ [
76
+ {
77
+ "name": "psi0_dance_gvhmr",
78
+ "root_dir": "$PSI_DANCE_MOTION_ROOT",
79
+ "video_dir": "$PSI_DANCE_VIDEO_ROOT",
80
+ "video_embed_dir": "$PSI_DANCE_VIDEO_EMBED_ROOT",
81
+ "dataset_name": "seed-video-g1"
82
+ },
83
+ {
84
+ "name": "motion_example_0506_demo",
85
+ "dataset_root": "$PSI_VIDEO_DEMO_ROOT",
86
+ "metadata_jsonl": "metadata.jsonl",
87
+ "motion_dir_key": "converted_path",
88
+ "video_path_key": "video_path",
89
+ "instruction_key": "video_caption",
90
+ "video_embed_dir": "$PSI_MOTION_EXAMPLE_VIDEO_EMBED_ROOT",
91
+ "dataset_name": "motion-example-video-g1"
92
+ }
93
+ ]
94
+ JSON
95
+ )
96
+ fi
97
+
98
+ if [ -z "$PSI_VIDEO_CLIP_SOURCES_JSON" ]; then
99
+ PSI_VIDEO_CLIP_SOURCES_JSON=$(compact_json <<JSON
100
+ [
101
+ {
102
+ "name": "dance_1723_video_10s_clip50",
103
+ "dataset_root": "$PSI_DANCE_VIDEO_CLIP_ROOT",
104
+ "metadata_jsonl": "metadata.jsonl",
105
+ "motion_dir_key": "converted_path",
106
+ "video_path_key": "video_path",
107
+ "instruction_key": "video_caption",
108
+ "video_embed_dir": "$PSI_DANCE_VIDEO_CLIP_ROOT/embeds",
109
+ "dataset_name": "dance-video-clip-g1"
110
+ },
111
+ {
112
+ "name": "motion_example_0506_demo_clip50",
113
+ "dataset_root": "$PSI_MOTION_EXAMPLE_VIDEO_CLIP_ROOT",
114
+ "metadata_jsonl": "metadata.jsonl",
115
+ "motion_dir_key": "converted_path",
116
+ "video_path_key": "video_path",
117
+ "instruction_key": "video_caption",
118
+ "video_embed_dir": "$PSI_MOTION_EXAMPLE_VIDEO_CLIP_ROOT/embeds",
119
+ "dataset_name": "motion-example-video-clip-g1"
120
+ }
121
+ ]
122
+ JSON
123
+ )
124
+ fi
125
+
126
+ PSI_TEXT_TASK_WEIGHT=0
127
+ PSI_AUDIO_TEXT_TASK_WEIGHT=0
128
+ PSI_AUDIO_ONLY_TASK_WEIGHT=0
129
+ PSI_AUDIO_COMMAND_TEXT_TASK_WEIGHT=0
130
+ PSI_AUDIO_COMMAND_ONLY_TASK_WEIGHT=0
131
+ PSI_VIDEO_TEXT_TASK_WEIGHT=0
132
+ PSI_VIDEO_ONLY_TASK_WEIGHT=0
133
+ PSI_VIDEO_CLIP_TEXT_TASK_WEIGHT=0.5
134
+ PSI_VIDEO_CLIP_ONLY_TASK_WEIGHT=0.5
135
+
136
+ PSI_PRETRAIN_BACKBONE_PATH=/cpfs01/VideoGen/public_ckpt/Qwen/Qwen2.5-Omni-3B
137
+ NUM_PAST_FRAMES=0
138
+ STATE_HORIZON=$((NUM_PAST_FRAMES + 1))
139
+ ACTION_CHUNK_SIZE=30
140
+ ACTION_EXEC_HORIZON=20
141
+ PROGRESS_DIM=5
142
+ VIDEO_SAMPLE_FPS=1.0
143
+ VIDEO_CLIP_SAMPLE_FPS=8.0
144
+ VIDEO_SIZE_PIXELS=50176
145
+ VIDEO_DECODER=ffmpeg
146
+ VIDEO_CACHE_SIZE=64
147
+ DATALOADER_NUM_WORKERS=4
148
+ VAL_DATALOADER_NUM_WORKERS=2
149
+
150
+ NPROC_PER_NODE=$(echo $CUDA_VISIBLE_DEVICES | tr ',' '\n' | wc -l)
151
+
152
+ echo "Training with $NPROC_PER_NODE GPU(s): $CUDA_VISIBLE_DEVICES"
153
+ echo "Text root: $PSI_SEED_ROOT"
154
+ echo "Dance motion root: $PSI_DANCE_MOTION_ROOT"
155
+ echo "Dance audio/video root: $PSI_DANCE_AUDIO_VIDEO_ROOT"
156
+ echo "Dance video root: $PSI_DANCE_VIDEO_ROOT"
157
+ echo "Dance video embed root: $PSI_DANCE_VIDEO_EMBED_ROOT"
158
+ echo "Qwen3-TTS acdudio command root: $PSI_QWEN3_TTS_ROOT"
159
+ echo "Audio sources: $PSI_AUDIO_SOURCES_JSON"
160
+ echo "Audio command sources: $PSI_AUDIO_COMMAND_SOURCES_JSON"
161
+ echo "Video sources: $PSI_VIDEO_SOURCES_JSON"
162
+ echo "Video clip sources: $PSI_VIDEO_CLIP_SOURCES_JSON"
163
+ echo "Task weights: text=$PSI_TEXT_TASK_WEIGHT audio_text=$PSI_AUDIO_TEXT_TASK_WEIGHT audio_only=$PSI_AUDIO_ONLY_TASK_WEIGHT audio_command_text=$PSI_AUDIO_COMMAND_TEXT_TASK_WEIGHT audio_command_only=$PSI_AUDIO_COMMAND_ONLY_TASK_WEIGHT video_text=$PSI_VIDEO_TEXT_TASK_WEIGHT video_only=$PSI_VIDEO_ONLY_TASK_WEIGHT video_clip_text=$PSI_VIDEO_CLIP_TEXT_TASK_WEIGHT video_clip_only=$PSI_VIDEO_CLIP_ONLY_TASK_WEIGHT"
164
+ # echo "Resume run dir: $PSI_PRETRAIN_RUN_DIR"
165
+ echo "TextOp 39D action/state enabled"
166
+ echo "Future progress embedding: enabled, dim $PROGRESS_DIM"
167
+ echo "Backbone path: $PSI_PRETRAIN_BACKBONE_PATH"
168
+ echo "Output dir: $PSI_OUTPUT_DIR"
169
+
170
+ args="posttrain_v1_seed_text_audio_video_mix_psi0_config \
171
+ --seed=292285 \
172
+ --exp=posttrain \
173
+ --timestamp=$(date +"%y%m%d%H%M") \
174
+ --train.name=posttrain_v1_trimodal \
175
+ --train.data_parallel=ddp \
176
+ --train.mixed_precision=bf16 \
177
+ --train.train_batch_size=32 \
178
+ --train.val_batch_size=64 \
179
+ --train.num-workers=$DATALOADER_NUM_WORKERS \
180
+ --train.val-num-workers=$VAL_DATALOADER_NUM_WORKERS \
181
+ --train.resume_from_checkpoint=None \
182
+ --train.max_checkpoints_to_keep=1 \
183
+ --train.gradient_accumulation_steps=1 \
184
+ --train.learning_rate=1e-4 \
185
+ --train.max_training_steps=500000 \
186
+ --train.warmup_ratio=None \
187
+ --train.warmup_steps=1000 \
188
+ --train.checkpointing_steps=1000 \
189
+ --train.validation_steps=1000 \
190
+ --train.val_num_batches=20 \
191
+ --train.max_grad_norm=1.0 \
192
+ --train.lr_scheduler_type=cosine_with_min_lr \
193
+ --train.lr_scheduler_kwargs.betas 0.9 0.999 \
194
+ --train.lr_scheduler_kwargs.weight_decay=0 \
195
+ --train.lr_scheduler_kwargs.eps=1e-8 \
196
+ --log.report_to=$PSI_LOG_REPORT_TO \
197
+ --train.output_dir=$PSI_OUTPUT_DIR \
198
+ --data.text.root-dir=$PSI_SEED_ROOT \
199
+ --data.text.metadata-csv=$PSI_SEED_METADATA \
200
+ --data.text-ratio=$PSI_TEXT_TASK_WEIGHT \
201
+ --data.audio.sources-json=$PSI_AUDIO_SOURCES_JSON \
202
+ --data.audio-ratio=$PSI_AUDIO_TEXT_TASK_WEIGHT \
203
+ --data.audio-only-ratio=$PSI_AUDIO_ONLY_TASK_WEIGHT \
204
+ --data.audio.sample-stride=1 \
205
+ --data.audio.motion-fps=50 \
206
+ --data.audio.audio-sampling-rate=16000 \
207
+ --data.audio.audio-window-seconds=1.0 \
208
+ --data.audio.audio-left-context-seconds=0.0 \
209
+ --data.audio.full-audio \
210
+ --data.audio-command.sources-json=$PSI_AUDIO_COMMAND_SOURCES_JSON \
211
+ --data.audio-command-ratio=$PSI_AUDIO_COMMAND_TEXT_TASK_WEIGHT \
212
+ --data.audio-command-only-ratio=$PSI_AUDIO_COMMAND_ONLY_TASK_WEIGHT \
213
+ --data.audio-command.sample-stride=1 \
214
+ --data.audio-command.motion-fps=50 \
215
+ --data.audio-command.audio-sampling-rate=16000 \
216
+ --data.audio-command.audio-window-seconds=1.0 \
217
+ --data.audio-command.audio-left-context-seconds=0.0 \
218
+ --data.audio-command.full-audio \
219
+ --data.video-ratio=$PSI_VIDEO_TEXT_TASK_WEIGHT \
220
+ --data.video-only-ratio=$PSI_VIDEO_ONLY_TASK_WEIGHT \
221
+ --data.video.sample-stride=1 \
222
+ --data.video.motion-fps=50 \
223
+ --data.video.video-sample-fps=$VIDEO_SAMPLE_FPS \
224
+ --data.video.video-size-pixels=$VIDEO_SIZE_PIXELS \
225
+ --data.video.video-decoder=$VIDEO_DECODER \
226
+ --data.video.video-cache-size=$VIDEO_CACHE_SIZE \
227
+ --data.video.sources-json=$PSI_VIDEO_SOURCES_JSON \
228
+ --data.video-clip-text-ratio=$PSI_VIDEO_CLIP_TEXT_TASK_WEIGHT \
229
+ --data.video-clip-only-ratio=$PSI_VIDEO_CLIP_ONLY_TASK_WEIGHT \
230
+ --data.video-clip.sources-json=$PSI_VIDEO_CLIP_SOURCES_JSON \
231
+ --data.video-clip.sample-stride=1 \
232
+ --data.video-clip.motion-fps=50 \
233
+ --data.video-clip.video-sample-fps=$VIDEO_CLIP_SAMPLE_FPS \
234
+ --data.video-clip.video-size-pixels=$VIDEO_SIZE_PIXELS \
235
+ --data.video-clip.video-decoder=$VIDEO_DECODER \
236
+ --data.video-clip.video-cache-size=$VIDEO_CACHE_SIZE \
237
+ --data.video-clip.action-exec-horizon=$ACTION_EXEC_HORIZON \
238
+ --data.sampler=batch_mixture \
239
+ --data.val-task=video_clip,video_clip_only \
240
+ --data.transform.repack.num-past-frames=$NUM_PAST_FRAMES \
241
+ --data.transform.repack.textop-actions \
242
+ --data.transform.repack.action-chunk-size=$ACTION_CHUNK_SIZE \
243
+ --data.transform.repack.pad-action-dim=39 \
244
+ --data.transform.repack.pad-state-dim=39 \
245
+ --data.transform.field.action_norm_type=bounds_q99 \
246
+ --data.transform.field.stat-path=assets/stats/seed_all_g1_motion39_textop_stats.json \
247
+ --data.transform.field.normalize-state \
248
+ --data.transform.field.pad-action-dim=39 \
249
+ --data.transform.field.pad-state-dim=39 \
250
+ --data.transform.model.resize.size 240 320 \
251
+ --data.transform.model.center_crop.size 240 320 \
252
+ --data.transform.model.no-img-aug \
253
+ --data.transform.model.text-only \
254
+ --data.transform.model.audio-only \
255
+ --data.transform.model.video-only \
256
+ --data.transform.model.video-size-pixels=$VIDEO_SIZE_PIXELS \
257
+ --model.model_name_or_path=$PSI_PRETRAIN_BACKBONE_PATH \
258
+ --model.noise-scheduler=flow \
259
+ --model.n_conditions=0 \
260
+ --model.action-chunk-size=$ACTION_CHUNK_SIZE \
261
+ --model.action-dim=39 \
262
+ --model.action-exec-horizon=$ACTION_EXEC_HORIZON \
263
+ --model.observation-horizon=$STATE_HORIZON \
264
+ --model.odim=39 \
265
+ --model.view_feature_dim=2048 \
266
+ --model.use-future-progress-embedding \
267
+ --model.future-progress-dim=$PROGRESS_DIM \
268
+ --model.tune-vlm \
269
+ --model.lang-backbone-lr=1e-5 \
270
+ --model.no-use_film \
271
+ --model.no-combined_temb \
272
+ --model.rtc \
273
+ --model.max-delay=11 \
274
+ --train.max-grad-value=1 \
275
+ --train.loss-group-weight.joint-abs=1.0 \
276
+ --train.loss-group-weight.root-delta-xy=3.0 \
277
+ --train.loss-group-weight.root-rot5-delta=10.0 \
278
+ --train.loss-group-weight.root-height=15.0 \
279
+ --train.loss-group-weight.foot-contact=0.1
280
+ "
281
+ torchrun --nproc_per_node=$NPROC_PER_NODE --master_port=29500 scripts/train.py \
282
+ $args "$@"
posttrain-v1-psi0-all.sh ADDED
@@ -0,0 +1,281 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ pkill -f monitor.py
2
+
3
+ cd /cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/Psi0-yth-v18
4
+ export PYTHONPATH=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/Psi0-yth-v18/src
5
+ export CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7
6
+ export PSI_OUTPUT_DIR=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/Psi0-yth-v18/runs
7
+ export PSI_LOG_REPORT_TO=wandb
8
+
9
+ source .venv-psi/bin/activate
10
+ [ -f .env ] && source .env
11
+
12
+ export WANDB_API_KEY="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"
13
+
14
+ export OMP_NUM_THREADS=8
15
+ export TF_CPP_MIN_LOG_LEVEL=3
16
+
17
+ # Keep JSON sources readable in this script, but pass compact JSON to tyro.
18
+ compact_json() {
19
+ python -c 'import json, sys; print(json.dumps(json.load(sys.stdin), separators=(",", ":")))'
20
+ }
21
+
22
+ PSI_SEED_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/Psi0/dataset/seed_all
23
+ PSI_SEED_METADATA=metadata/seed_metadata_v004.csv
24
+
25
+ PSI_DANCE_MOTION_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/GR00T-WholeBodyControl/gear_sonic_deploy/reference/converted/psi0_dance/gvhmr
26
+ PSI_DANCE_AUDIO_VIDEO_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/dance_1723_video_psi0/dance_1723_video_10s
27
+ PSI_DANCE_VIDEO_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/dance_1723_video_psi0/dance_1723_video_10s
28
+ PSI_DANCE_VIDEO_EMBED_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/dance_1723_video_psi0/dance_1723_video_10s_embeds
29
+ PSI_VIDEO_DEMO_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/videogen/motion_example_0506_demo
30
+ PSI_QWEN3_TTS_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/qwen3-tts
31
+ PSI_VIDEO_CLIP_DATASET_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/videogen/psi0_video_clip_50f_rand8_full
32
+ PSI_DANCE_VIDEO_CLIP_ROOT=$PSI_VIDEO_CLIP_DATASET_ROOT/dance_1723_video_10s
33
+ PSI_MOTION_EXAMPLE_VIDEO_CLIP_ROOT=$PSI_VIDEO_CLIP_DATASET_ROOT/motion_example_0506_demo
34
+ PSI_MOTION_EXAMPLE_VIDEO_EMBED_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/videogen/motion_example_0506_demo_video_embeds
35
+
36
+ if [ -z "$PSI_AUDIO_SOURCES_JSON" ]; then
37
+ PSI_AUDIO_SOURCES_JSON=$(compact_json <<JSON
38
+ [
39
+ {
40
+ "name": "psi0_dance_audio",
41
+ "root_dir": "$PSI_DANCE_MOTION_ROOT",
42
+ "video_dir": "$PSI_DANCE_AUDIO_VIDEO_ROOT",
43
+ "dataset_name": "seed-audio-g1"
44
+ }
45
+ ]
46
+ JSON
47
+ )
48
+ fi
49
+
50
+ if [ -z "$PSI_AUDIO_COMMAND_SOURCES_JSON" ]; then
51
+ PSI_AUDIO_COMMAND_SOURCES_JSON=$(compact_json <<JSON
52
+ [
53
+ {
54
+ "name": "qwen3_tts_seed_audio_command",
55
+ "root_dir": "$PSI_SEED_ROOT",
56
+ "dataset_root": "$PSI_QWEN3_TTS_ROOT",
57
+ "metadata_jsonl": "metadata.jsonl",
58
+ "motion_dir_key": "filename",
59
+ "audio_path_key": "audio_path",
60
+ "instruction_key": "content_short_description",
61
+ "motion_metadata_csv": "$PSI_SEED_METADATA",
62
+ "motion_metadata_key": "filename",
63
+ "motion_metadata_path_key": "move_g1_path",
64
+ "audio_file_extensions": ".wav",
65
+ "audio_task": "command",
66
+ "dataset_name": "seed-audio-command-g1"
67
+ }
68
+ ]
69
+ JSON
70
+ )
71
+ fi
72
+
73
+ if [ -z "$PSI_VIDEO_SOURCES_JSON" ]; then
74
+ PSI_VIDEO_SOURCES_JSON=$(compact_json <<JSON
75
+ [
76
+ {
77
+ "name": "psi0_dance_gvhmr",
78
+ "root_dir": "$PSI_DANCE_MOTION_ROOT",
79
+ "video_dir": "$PSI_DANCE_VIDEO_ROOT",
80
+ "video_embed_dir": "$PSI_DANCE_VIDEO_EMBED_ROOT",
81
+ "dataset_name": "seed-video-g1"
82
+ },
83
+ {
84
+ "name": "motion_example_0506_demo",
85
+ "dataset_root": "$PSI_VIDEO_DEMO_ROOT",
86
+ "metadata_jsonl": "metadata.jsonl",
87
+ "motion_dir_key": "converted_path",
88
+ "video_path_key": "video_path",
89
+ "instruction_key": "video_caption",
90
+ "video_embed_dir": "$PSI_MOTION_EXAMPLE_VIDEO_EMBED_ROOT",
91
+ "dataset_name": "motion-example-video-g1"
92
+ }
93
+ ]
94
+ JSON
95
+ )
96
+ fi
97
+
98
+ if [ -z "$PSI_VIDEO_CLIP_SOURCES_JSON" ]; then
99
+ PSI_VIDEO_CLIP_SOURCES_JSON=$(compact_json <<JSON
100
+ [
101
+ {
102
+ "name": "dance_1723_video_10s_clip50",
103
+ "dataset_root": "$PSI_DANCE_VIDEO_CLIP_ROOT",
104
+ "metadata_jsonl": "metadata.jsonl",
105
+ "motion_dir_key": "converted_path",
106
+ "video_path_key": "video_path",
107
+ "instruction_key": "video_caption",
108
+ "video_embed_dir": "$PSI_DANCE_VIDEO_CLIP_ROOT/embeds",
109
+ "dataset_name": "dance-video-clip-g1"
110
+ },
111
+ {
112
+ "name": "motion_example_0506_demo_clip50",
113
+ "dataset_root": "$PSI_MOTION_EXAMPLE_VIDEO_CLIP_ROOT",
114
+ "metadata_jsonl": "metadata.jsonl",
115
+ "motion_dir_key": "converted_path",
116
+ "video_path_key": "video_path",
117
+ "instruction_key": "video_caption",
118
+ "video_embed_dir": "$PSI_MOTION_EXAMPLE_VIDEO_CLIP_ROOT/embeds",
119
+ "dataset_name": "motion-example-video-clip-g1"
120
+ }
121
+ ]
122
+ JSON
123
+ )
124
+ fi
125
+
126
+ PSI_TEXT_TASK_WEIGHT=1
127
+ PSI_AUDIO_TEXT_TASK_WEIGHT=0
128
+ PSI_AUDIO_ONLY_TASK_WEIGHT=0
129
+ PSI_AUDIO_COMMAND_TEXT_TASK_WEIGHT=0
130
+ PSI_AUDIO_COMMAND_ONLY_TASK_WEIGHT=0
131
+ PSI_VIDEO_TEXT_TASK_WEIGHT=0
132
+ PSI_VIDEO_ONLY_TASK_WEIGHT=0
133
+ PSI_VIDEO_CLIP_TEXT_TASK_WEIGHT=0
134
+ PSI_VIDEO_CLIP_ONLY_TASK_WEIGHT=0
135
+
136
+ PSI_PRETRAIN_BACKBONE_PATH=/cpfs01/VideoGen/public_ckpt/Qwen/Qwen2.5-Omni-3B
137
+ NUM_PAST_FRAMES=0
138
+ STATE_HORIZON=$((NUM_PAST_FRAMES + 1))
139
+ ACTION_CHUNK_SIZE=30
140
+ ACTION_EXEC_HORIZON=20
141
+ PROGRESS_DIM=5
142
+ VIDEO_SAMPLE_FPS=1.0
143
+ VIDEO_CLIP_SAMPLE_FPS=8.0
144
+ VIDEO_SIZE_PIXELS=50176
145
+ VIDEO_DECODER=ffmpeg
146
+ VIDEO_CACHE_SIZE=64
147
+ DATALOADER_NUM_WORKERS=4
148
+ VAL_DATALOADER_NUM_WORKERS=2
149
+
150
+ NPROC_PER_NODE=$(echo $CUDA_VISIBLE_DEVICES | tr ',' '\n' | wc -l)
151
+
152
+ echo "Training with $NPROC_PER_NODE GPU(s): $CUDA_VISIBLE_DEVICES"
153
+ echo "Text root: $PSI_SEED_ROOT"
154
+ echo "Dance motion root: $PSI_DANCE_MOTION_ROOT"
155
+ echo "Dance audio/video root: $PSI_DANCE_AUDIO_VIDEO_ROOT"
156
+ echo "Dance video root: $PSI_DANCE_VIDEO_ROOT"
157
+ echo "Dance video embed root: $PSI_DANCE_VIDEO_EMBED_ROOT"
158
+ echo "Qwen3-TTS acdudio command root: $PSI_QWEN3_TTS_ROOT"
159
+ echo "Audio sources: $PSI_AUDIO_SOURCES_JSON"
160
+ echo "Audio command sources: $PSI_AUDIO_COMMAND_SOURCES_JSON"
161
+ echo "Video sources: $PSI_VIDEO_SOURCES_JSON"
162
+ echo "Video clip sources: $PSI_VIDEO_CLIP_SOURCES_JSON"
163
+ echo "Task weights: text=$PSI_TEXT_TASK_WEIGHT audio_text=$PSI_AUDIO_TEXT_TASK_WEIGHT audio_only=$PSI_AUDIO_ONLY_TASK_WEIGHT audio_command_text=$PSI_AUDIO_COMMAND_TEXT_TASK_WEIGHT audio_command_only=$PSI_AUDIO_COMMAND_ONLY_TASK_WEIGHT video_text=$PSI_VIDEO_TEXT_TASK_WEIGHT video_only=$PSI_VIDEO_ONLY_TASK_WEIGHT video_clip_text=$PSI_VIDEO_CLIP_TEXT_TASK_WEIGHT video_clip_only=$PSI_VIDEO_CLIP_ONLY_TASK_WEIGHT"
164
+ # echo "Resume run dir: $PSI_PRETRAIN_RUN_DIR"
165
+ echo "TextOp 39D action/state enabled"
166
+ echo "Future progress embedding: enabled, dim $PROGRESS_DIM"
167
+ echo "Backbone path: $PSI_PRETRAIN_BACKBONE_PATH"
168
+ echo "Output dir: $PSI_OUTPUT_DIR"
169
+
170
+ args="posttrain_v1_seed_text_audio_video_mix_psi0_config \
171
+ --seed=292285 \
172
+ --exp=posttrain \
173
+ --timestamp=$(date +"%y%m%d%H%M") \
174
+ --train.name=posttrain_v1_trimodal \
175
+ --train.data_parallel=ddp \
176
+ --train.mixed_precision=bf16 \
177
+ --train.train_batch_size=64 \
178
+ --train.val_batch_size=64 \
179
+ --train.num-workers=$DATALOADER_NUM_WORKERS \
180
+ --train.val-num-workers=$VAL_DATALOADER_NUM_WORKERS \
181
+ --train.resume_from_checkpoint=None \
182
+ --train.max_checkpoints_to_keep=1 \
183
+ --train.gradient_accumulation_steps=1 \
184
+ --train.learning_rate=1e-4 \
185
+ --train.max_training_steps=1000000 \
186
+ --train.warmup_ratio=None \
187
+ --train.warmup_steps=1000 \
188
+ --train.checkpointing_steps=1000 \
189
+ --train.validation_steps=1000 \
190
+ --train.val_num_batches=20 \
191
+ --train.max_grad_norm=1.0 \
192
+ --train.lr_scheduler_type=cosine_with_min_lr \
193
+ --train.lr_scheduler_kwargs.betas 0.9 0.999 \
194
+ --train.lr_scheduler_kwargs.weight_decay=0 \
195
+ --train.lr_scheduler_kwargs.eps=1e-8 \
196
+ --log.report_to=$PSI_LOG_REPORT_TO \
197
+ --train.output_dir=$PSI_OUTPUT_DIR \
198
+ --data.text.root-dir=$PSI_SEED_ROOT \
199
+ --data.text.metadata-csv=$PSI_SEED_METADATA \
200
+ --data.text-ratio=$PSI_TEXT_TASK_WEIGHT \
201
+ --data.audio.sources-json=$PSI_AUDIO_SOURCES_JSON \
202
+ --data.audio-ratio=$PSI_AUDIO_TEXT_TASK_WEIGHT \
203
+ --data.audio-only-ratio=$PSI_AUDIO_ONLY_TASK_WEIGHT \
204
+ --data.audio.sample-stride=1 \
205
+ --data.audio.motion-fps=50 \
206
+ --data.audio.audio-sampling-rate=16000 \
207
+ --data.audio.audio-window-seconds=1.0 \
208
+ --data.audio.audio-left-context-seconds=0.0 \
209
+ --data.audio.full-audio \
210
+ --data.audio-command.sources-json=$PSI_AUDIO_COMMAND_SOURCES_JSON \
211
+ --data.audio-command-ratio=$PSI_AUDIO_COMMAND_TEXT_TASK_WEIGHT \
212
+ --data.audio-command-only-ratio=$PSI_AUDIO_COMMAND_ONLY_TASK_WEIGHT \
213
+ --data.audio-command.sample-stride=1 \
214
+ --data.audio-command.motion-fps=50 \
215
+ --data.audio-command.audio-sampling-rate=16000 \
216
+ --data.audio-command.audio-window-seconds=1.0 \
217
+ --data.audio-command.audio-left-context-seconds=0.0 \
218
+ --data.audio-command.full-audio \
219
+ --data.video-ratio=$PSI_VIDEO_TEXT_TASK_WEIGHT \
220
+ --data.video-only-ratio=$PSI_VIDEO_ONLY_TASK_WEIGHT \
221
+ --data.video.sample-stride=1 \
222
+ --data.video.motion-fps=50 \
223
+ --data.video.video-sample-fps=$VIDEO_SAMPLE_FPS \
224
+ --data.video.video-size-pixels=$VIDEO_SIZE_PIXELS \
225
+ --data.video.video-decoder=$VIDEO_DECODER \
226
+ --data.video.video-cache-size=$VIDEO_CACHE_SIZE \
227
+ --data.video.sources-json=$PSI_VIDEO_SOURCES_JSON \
228
+ --data.video-clip-text-ratio=$PSI_VIDEO_CLIP_TEXT_TASK_WEIGHT \
229
+ --data.video-clip-only-ratio=$PSI_VIDEO_CLIP_ONLY_TASK_WEIGHT \
230
+ --data.video-clip.sources-json=$PSI_VIDEO_CLIP_SOURCES_JSON \
231
+ --data.video-clip.sample-stride=1 \
232
+ --data.video-clip.motion-fps=50 \
233
+ --data.video-clip.video-sample-fps=$VIDEO_CLIP_SAMPLE_FPS \
234
+ --data.video-clip.video-size-pixels=$VIDEO_SIZE_PIXELS \
235
+ --data.video-clip.video-decoder=$VIDEO_DECODER \
236
+ --data.video-clip.video-cache-size=$VIDEO_CACHE_SIZE \
237
+ --data.video-clip.action-exec-horizon=$ACTION_EXEC_HORIZON \
238
+ --data.sampler=batch_mixture \
239
+ --data.val-task=text \
240
+ --data.transform.repack.num-past-frames=$NUM_PAST_FRAMES \
241
+ --data.transform.repack.textop-actions \
242
+ --data.transform.repack.action-chunk-size=$ACTION_CHUNK_SIZE \
243
+ --data.transform.repack.pad-action-dim=39 \
244
+ --data.transform.repack.pad-state-dim=39 \
245
+ --data.transform.field.action_norm_type=bounds_q99 \
246
+ --data.transform.field.stat-path=assets/stats/seed_all_g1_motion39_textop_stats.json \
247
+ --data.transform.field.normalize-state \
248
+ --data.transform.field.pad-action-dim=39 \
249
+ --data.transform.field.pad-state-dim=39 \
250
+ --data.transform.model.resize.size 240 320 \
251
+ --data.transform.model.center_crop.size 240 320 \
252
+ --data.transform.model.no-img-aug \
253
+ --data.transform.model.text-only \
254
+ --data.transform.model.audio-only \
255
+ --data.transform.model.video-only \
256
+ --data.transform.model.video-size-pixels=$VIDEO_SIZE_PIXELS \
257
+ --model.model_name_or_path=$PSI_PRETRAIN_BACKBONE_PATH \
258
+ --model.noise-scheduler=flow \
259
+ --model.n_conditions=0 \
260
+ --model.action-chunk-size=$ACTION_CHUNK_SIZE \
261
+ --model.action-dim=39 \
262
+ --model.action-exec-horizon=$ACTION_EXEC_HORIZON \
263
+ --model.observation-horizon=$STATE_HORIZON \
264
+ --model.odim=39 \
265
+ --model.view_feature_dim=2048 \
266
+ --model.use-future-progress-embedding \
267
+ --model.future-progress-dim=$PROGRESS_DIM \
268
+ --model.no-tune-vlm \
269
+ --model.no-use_film \
270
+ --model.no-combined_temb \
271
+ --model.rtc \
272
+ --model.max-delay=11 \
273
+ --train.max-grad-value=1 \
274
+ --train.loss-group-weight.joint-abs=1.0 \
275
+ --train.loss-group-weight.root-delta-xy=1.0 \
276
+ --train.loss-group-weight.root-rot5-delta=1.0 \
277
+ --train.loss-group-weight.root-height=1.0 \
278
+ --train.loss-group-weight.foot-contact=1.0
279
+ "
280
+ torchrun --nproc_per_node=$NPROC_PER_NODE --master_port=29500 scripts/train.py \
281
+ $args "$@"
posttrain-v1-psi0-all_resume.sh ADDED
@@ -0,0 +1,285 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ pkill -f monitor.py
2
+
3
+ cd /cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/Psi0-yth-v18
4
+ export PYTHONPATH=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/Psi0-yth-v18/src
5
+ export CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7
6
+ export PSI_OUTPUT_DIR=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/Psi0-yth-v18/runs
7
+ export PSI_LOG_REPORT_TO=wandb
8
+
9
+ source .venv-psi/bin/activate
10
+ [ -f .env ] && source .env
11
+
12
+ export WANDB_API_KEY="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"
13
+
14
+ export OMP_NUM_THREADS=${OMP_NUM_THREADS:-1}
15
+ export TF_CPP_MIN_LOG_LEVEL=3
16
+
17
+ # Keep JSON sources readable in this script, but pass compact JSON to tyro.
18
+ compact_json() {
19
+ python -c 'import json, sys; print(json.dumps(json.load(sys.stdin), separators=(",", ":")))'
20
+ }
21
+
22
+ PSI_SEED_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/Psi0/dataset/seed_all
23
+ PSI_SEED_METADATA=metadata/seed_metadata_v004.csv
24
+
25
+ PSI_DANCE_MOTION_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/GR00T-WholeBodyControl/gear_sonic_deploy/reference/converted/psi0_dance/gvhmr
26
+ PSI_DANCE_AUDIO_VIDEO_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/dance_1723_video_psi0/dance_1723_video_10s
27
+ PSI_DANCE_VIDEO_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/dance_1723_video_psi0/dance_1723_video_10s
28
+ PSI_DANCE_VIDEO_EMBED_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/dance_1723_video_psi0/dance_1723_video_10s_embeds
29
+ PSI_VIDEO_DEMO_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/videogen/motion_example_0506_demo
30
+ PSI_QWEN3_TTS_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/qwen3-tts
31
+ PSI_VIDEO_CLIP_DATASET_ROOT=${PSI_VIDEO_CLIP_DATASET_ROOT:-/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/videogen/psi0_video_clip_50f_rand32_full}
32
+ PSI_DANCE_VIDEO_CLIP_ROOT=$PSI_VIDEO_CLIP_DATASET_ROOT/dance_1723_video_10s
33
+ PSI_MOTION_EXAMPLE_VIDEO_CLIP_ROOT=$PSI_VIDEO_CLIP_DATASET_ROOT/motion_example_0506_demo
34
+ PSI_MOTION_EXAMPLE_VIDEO_EMBED_ROOT=${PSI_MOTION_EXAMPLE_VIDEO_EMBED_ROOT:-/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/videogen/motion_example_0506_demo_video_embeds}
35
+
36
+ if [ -z "${PSI_AUDIO_SOURCES_JSON:-}" ]; then
37
+ PSI_AUDIO_SOURCES_JSON=$(compact_json <<JSON
38
+ [
39
+ {
40
+ "name": "psi0_dance_audio",
41
+ "root_dir": "$PSI_DANCE_MOTION_ROOT",
42
+ "video_dir": "$PSI_DANCE_AUDIO_VIDEO_ROOT",
43
+ "dataset_name": "seed-audio-g1"
44
+ }
45
+ ]
46
+ JSON
47
+ )
48
+ fi
49
+
50
+ if [ -z "${PSI_AUDIO_COMMAND_SOURCES_JSON:-}" ]; then
51
+ PSI_AUDIO_COMMAND_SOURCES_JSON=$(compact_json <<JSON
52
+ [
53
+ {
54
+ "name": "qwen3_tts_seed_audio_command",
55
+ "root_dir": "$PSI_SEED_ROOT",
56
+ "dataset_root": "$PSI_QWEN3_TTS_ROOT",
57
+ "metadata_jsonl": "metadata.jsonl",
58
+ "motion_dir_key": "filename",
59
+ "audio_path_key": "audio_path",
60
+ "instruction_key": "content_short_description",
61
+ "motion_metadata_csv": "$PSI_SEED_METADATA",
62
+ "motion_metadata_key": "filename",
63
+ "motion_metadata_path_key": "move_g1_path",
64
+ "audio_file_extensions": ".wav",
65
+ "audio_task": "command",
66
+ "dataset_name": "seed-audio-command-g1"
67
+ }
68
+ ]
69
+ JSON
70
+ )
71
+ fi
72
+
73
+ if [ -z "${PSI_VIDEO_SOURCES_JSON:-}" ]; then
74
+ PSI_VIDEO_SOURCES_JSON=$(compact_json <<JSON
75
+ [
76
+ {
77
+ "name": "psi0_dance_gvhmr",
78
+ "root_dir": "$PSI_DANCE_MOTION_ROOT",
79
+ "video_dir": "$PSI_DANCE_VIDEO_ROOT",
80
+ "video_embed_dir": "$PSI_DANCE_VIDEO_EMBED_ROOT",
81
+ "dataset_name": "seed-video-g1"
82
+ },
83
+ {
84
+ "name": "motion_example_0506_demo",
85
+ "dataset_root": "$PSI_VIDEO_DEMO_ROOT",
86
+ "metadata_jsonl": "metadata.jsonl",
87
+ "motion_dir_key": "converted_path",
88
+ "video_path_key": "video_path",
89
+ "instruction_key": "video_caption",
90
+ "video_embed_dir": "$PSI_MOTION_EXAMPLE_VIDEO_EMBED_ROOT",
91
+ "dataset_name": "motion-example-video-g1"
92
+ }
93
+ ]
94
+ JSON
95
+ )
96
+ fi
97
+
98
+
99
+
100
+
101
+ if [ -z "${PSI_VIDEO_CLIP_SOURCES_JSON:-}" ]; then
102
+ PSI_VIDEO_CLIP_SOURCES_JSON=$(compact_json <<JSON
103
+ [
104
+ {
105
+ "name": "dance_1723_video_10s_clip50",
106
+ "dataset_root": "$PSI_DANCE_VIDEO_CLIP_ROOT",
107
+ "metadata_jsonl": "metadata.jsonl",
108
+ "motion_dir_key": "converted_path",
109
+ "video_path_key": "video_path",
110
+ "instruction_key": "video_caption",
111
+ "video_embed_dir": "$PSI_DANCE_VIDEO_CLIP_ROOT/embeds",
112
+ "dataset_name": "dance-video-clip-g1"
113
+ },
114
+ {
115
+ "name": "motion_example_0506_demo_clip50",
116
+ "dataset_root": "$PSI_MOTION_EXAMPLE_VIDEO_CLIP_ROOT",
117
+ "metadata_jsonl": "metadata.jsonl",
118
+ "motion_dir_key": "converted_path",
119
+ "video_path_key": "video_path",
120
+ "instruction_key": "video_caption",
121
+ "video_embed_dir": "$PSI_MOTION_EXAMPLE_VIDEO_CLIP_ROOT/embeds",
122
+ "dataset_name": "motion-example-video-clip-g1"
123
+ }
124
+ ]
125
+ JSON
126
+ )
127
+ fi
128
+
129
+
130
+
131
+ PSI_TEXT_TASK_WEIGHT=1
132
+ PSI_AUDIO_TEXT_TASK_WEIGHT=0.5
133
+ PSI_AUDIO_ONLY_TASK_WEIGHT=0.5
134
+ PSI_AUDIO_COMMAND_TEXT_TASK_WEIGHT=0.5
135
+ PSI_AUDIO_COMMAND_ONLY_TASK_WEIGHT=0.5
136
+ PSI_VIDEO_TEXT_TASK_WEIGHT=0.5
137
+ PSI_VIDEO_ONLY_TASK_WEIGHT=0.5
138
+ PSI_VIDEO_CLIP_TEXT_TASK_WEIGHT=0.5
139
+ PSI_VIDEO_CLIP_ONLY_TASK_WEIGHT=0.5
140
+
141
+ PSI_PRETRAIN_BACKBONE_PATH=/cpfs01/VideoGen/public_ckpt/Qwen/Qwen2.5-Omni-3B
142
+ # PSI_PRETRAIN_RUN_DIR=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/Psi0-yth-v9/runs/posttrain_v1/posttrain.sg.flow1000.cwml.lr1.0e-04.b512.gpus8.2604250426
143
+ PSI_PRETRAIN_RUN_DIR=${PSI_PRETRAIN_RUN_DIR:-/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/Psi0-yth-v12/runs/posttrain_v1_trimodal/posttrain.sg.flow1000.cwml.lr1.0e-04.b512.gpus8.2605140518}
144
+
145
+ NUM_PAST_FRAMES=0
146
+ STATE_HORIZON=$((NUM_PAST_FRAMES + 1))
147
+ ACTION_CHUNK_SIZE=30
148
+ ACTION_EXEC_HORIZON=20
149
+ VIDEO_SAMPLE_FPS=1.0
150
+ VIDEO_CLIP_SAMPLE_FPS=${VIDEO_CLIP_SAMPLE_FPS:-32.0}
151
+ VIDEO_SIZE_PIXELS=50176
152
+ VIDEO_DECODER=${VIDEO_DECODER:-ffmpeg}
153
+ VIDEO_CACHE_SIZE=${VIDEO_CACHE_SIZE:-64}
154
+ DATALOADER_NUM_WORKERS=${DATALOADER_NUM_WORKERS:-4}
155
+ VAL_DATALOADER_NUM_WORKERS=${VAL_DATALOADER_NUM_WORKERS:-2}
156
+
157
+ NPROC_PER_NODE=$(echo $CUDA_VISIBLE_DEVICES | tr ',' '\n' | wc -l)
158
+
159
+ echo "Training with $NPROC_PER_NODE GPU(s): $CUDA_VISIBLE_DEVICES"
160
+ echo "Text root: $PSI_SEED_ROOT"
161
+ echo "Dance motion root: $PSI_DANCE_MOTION_ROOT"
162
+ echo "Dance audio/video root: $PSI_DANCE_AUDIO_VIDEO_ROOT"
163
+ echo "Dance video root: $PSI_DANCE_VIDEO_ROOT"
164
+ echo "Dance video embed root: $PSI_DANCE_VIDEO_EMBED_ROOT"
165
+ echo "Qwen3-TTS audio command root: $PSI_QWEN3_TTS_ROOT"
166
+ echo "Audio sources: $PSI_AUDIO_SOURCES_JSON"
167
+ echo "Audio command sources: $PSI_AUDIO_COMMAND_SOURCES_JSON"
168
+ echo "Video sources: $PSI_VIDEO_SOURCES_JSON"
169
+ echo "Video clip sources: $PSI_VIDEO_CLIP_SOURCES_JSON"
170
+ echo "Task weights: text=$PSI_TEXT_TASK_WEIGHT audio_text=$PSI_AUDIO_TEXT_TASK_WEIGHT audio_only=$PSI_AUDIO_ONLY_TASK_WEIGHT audio_command_text=$PSI_AUDIO_COMMAND_TEXT_TASK_WEIGHT audio_command_only=$PSI_AUDIO_COMMAND_ONLY_TASK_WEIGHT video_text=$PSI_VIDEO_TEXT_TASK_WEIGHT video_only=$PSI_VIDEO_ONLY_TASK_WEIGHT video_clip_text=$PSI_VIDEO_CLIP_TEXT_TASK_WEIGHT video_clip_only=$PSI_VIDEO_CLIP_ONLY_TASK_WEIGHT"
171
+ echo "Resume run dir: $PSI_PRETRAIN_RUN_DIR"
172
+ echo "Backbone path: $PSI_PRETRAIN_BACKBONE_PATH"
173
+ echo "Output dir: $PSI_OUTPUT_DIR"
174
+
175
+ args="posttrain_v1_seed_text_audio_video_mix_psi0_config \
176
+ --seed=292285 \
177
+ --exp=posttrain \
178
+ --timestamp=$(date +"%y%m%d%H%M") \
179
+ --train.name=posttrain_v1_trimodal \
180
+ --train.data_parallel=ddp \
181
+ --train.mixed_precision=bf16 \
182
+ --train.train_batch_size=64 \
183
+ --train.val_batch_size=64 \
184
+ --train.num-workers=$DATALOADER_NUM_WORKERS \
185
+ --train.val-num-workers=$VAL_DATALOADER_NUM_WORKERS \
186
+ --train.resume_from_checkpoint=$PSI_PRETRAIN_RUN_DIR \
187
+ --train.max_checkpoints_to_keep=1 \
188
+ --train.gradient_accumulation_steps=1 \
189
+ --train.learning_rate=1e-4 \
190
+ --train.max_training_steps=1000000 \
191
+ --train.warmup_ratio=None \
192
+ --train.warmup_steps=1000 \
193
+ --train.checkpointing_steps=1000 \
194
+ --train.validation_steps=1000 \
195
+ --train.val_num_batches=20 \
196
+ --train.max_grad_norm=1.0 \
197
+ --train.lr_scheduler_type=cosine_with_min_lr \
198
+ --train.lr_scheduler_kwargs.betas 0.9 0.999 \
199
+ --train.lr_scheduler_kwargs.weight_decay=0 \
200
+ --train.lr_scheduler_kwargs.eps=1e-8 \
201
+ --log.report_to=$PSI_LOG_REPORT_TO \
202
+ --train.output_dir=$PSI_OUTPUT_DIR \
203
+ --data.text.root-dir=$PSI_SEED_ROOT \
204
+ --data.text.metadata-csv=$PSI_SEED_METADATA \
205
+ --data.text-ratio=$PSI_TEXT_TASK_WEIGHT \
206
+ --data.audio.sources-json=$PSI_AUDIO_SOURCES_JSON \
207
+ --data.audio-ratio=$PSI_AUDIO_TEXT_TASK_WEIGHT \
208
+ --data.audio-only-ratio=$PSI_AUDIO_ONLY_TASK_WEIGHT \
209
+ --data.audio.sample-stride=1 \
210
+ --data.audio.motion-fps=50 \
211
+ --data.audio.audio-sampling-rate=16000 \
212
+ --data.audio.audio-window-seconds=1.0 \
213
+ --data.audio.audio-left-context-seconds=0.0 \
214
+ --data.audio.full-audio \
215
+ --data.audio-command.sources-json=$PSI_AUDIO_COMMAND_SOURCES_JSON \
216
+ --data.audio-command-ratio=$PSI_AUDIO_COMMAND_TEXT_TASK_WEIGHT \
217
+ --data.audio-command-only-ratio=$PSI_AUDIO_COMMAND_ONLY_TASK_WEIGHT \
218
+ --data.audio-command.sample-stride=1 \
219
+ --data.audio-command.motion-fps=50 \
220
+ --data.audio-command.audio-sampling-rate=16000 \
221
+ --data.audio-command.audio-window-seconds=1.0 \
222
+ --data.audio-command.audio-left-context-seconds=0.0 \
223
+ --data.audio-command.full-audio \
224
+ --data.video-ratio=$PSI_VIDEO_TEXT_TASK_WEIGHT \
225
+ --data.video-only-ratio=$PSI_VIDEO_ONLY_TASK_WEIGHT \
226
+ --data.video.sample-stride=1 \
227
+ --data.video.motion-fps=50 \
228
+ --data.video.video-sample-fps=$VIDEO_SAMPLE_FPS \
229
+ --data.video.video-size-pixels=$VIDEO_SIZE_PIXELS \
230
+ --data.video.video-decoder=$VIDEO_DECODER \
231
+ --data.video.video-cache-size=$VIDEO_CACHE_SIZE \
232
+ --data.video.sources-json=$PSI_VIDEO_SOURCES_JSON \
233
+ --data.video-clip-text-ratio=$PSI_VIDEO_CLIP_TEXT_TASK_WEIGHT \
234
+ --data.video-clip-only-ratio=$PSI_VIDEO_CLIP_ONLY_TASK_WEIGHT \
235
+ --data.video-clip.sources-json=$PSI_VIDEO_CLIP_SOURCES_JSON \
236
+ --data.video-clip.sample-stride=1 \
237
+ --data.video-clip.motion-fps=50 \
238
+ --data.video-clip.video-sample-fps=$VIDEO_CLIP_SAMPLE_FPS \
239
+ --data.video-clip.video-size-pixels=$VIDEO_SIZE_PIXELS \
240
+ --data.video-clip.video-decoder=$VIDEO_DECODER \
241
+ --data.video-clip.video-cache-size=$VIDEO_CACHE_SIZE \
242
+ --data.video-clip.action-exec-horizon=$ACTION_EXEC_HORIZON \
243
+ --data.sampler=batch_mixture \
244
+ --data.val-task=text,audio,audio_only,audio_command,audio_command_only,video,video_only,video_clip,video_clip_only \
245
+ --data.transform.repack.num-past-frames=$NUM_PAST_FRAMES \
246
+ --data.transform.repack.textop-actions \
247
+ --data.transform.repack.action-chunk-size=$ACTION_CHUNK_SIZE \
248
+ --data.transform.repack.pad-action-dim=70 \
249
+ --data.transform.repack.pad-state-dim=36 \
250
+ --data.transform.field.action_norm_type=bounds_q99 \
251
+ --data.transform.field.stat-path=assets/stats/seed_all_g1_motion70_textop_stats.json \
252
+ --data.transform.field.normalize-state \
253
+ --data.transform.field.pad-action-dim=70 \
254
+ --data.transform.field.pad-state-dim=36 \
255
+ --data.transform.model.resize.size 240 320 \
256
+ --data.transform.model.center_crop.size 240 320 \
257
+ --data.transform.model.no-img-aug \
258
+ --data.transform.model.text-only \
259
+ --data.transform.model.audio-only \
260
+ --data.transform.model.video-only \
261
+ --data.transform.model.video-size-pixels=$VIDEO_SIZE_PIXELS \
262
+ --model.model_name_or_path=$PSI_PRETRAIN_BACKBONE_PATH \
263
+ --model.noise-scheduler=flow \
264
+ --model.n_conditions=0 \
265
+ --model.action-chunk-size=$ACTION_CHUNK_SIZE \
266
+ --model.action-dim=70 \
267
+ --model.action-exec-horizon=$ACTION_EXEC_HORIZON \
268
+ --model.observation-horizon=$STATE_HORIZON \
269
+ --model.odim=36 \
270
+ --model.view_feature_dim=2048 \
271
+ --model.no-tune-vlm \
272
+ --model.no-use_film \
273
+ --model.no-combined_temb \
274
+ --model.rtc \
275
+ --model.max-delay=11 \
276
+ --train.max-grad-value=1 \
277
+ --train.loss-group-weight.joint-abs=0.1 \
278
+ --train.loss-group-weight.joint-delta=1.0 \
279
+ --train.loss-group-weight.root-delta-local=1.0 \
280
+ --train.loss-group-weight.root-rot6d-delta=1.0 \
281
+ --train.loss-group-weight.root-height=1.0 \
282
+ --train.loss-group-weight.foot-contact=1.0
283
+ "
284
+ torchrun --nproc_per_node=$NPROC_PER_NODE --master_port=29500 scripts/train.py \
285
+ ${args} "$@"
posttrain-v1-psi0-audio-clip-tune-vlm.sh ADDED
@@ -0,0 +1,282 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ pkill -f monitor.py
2
+
3
+ cd /cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/Psi0-yth-v18
4
+ export PYTHONPATH=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/Psi0-yth-v18/src
5
+ export CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7
6
+ export PSI_OUTPUT_DIR=${PSI_OUTPUT_DIR:-/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/Psi0-yth-v18/runs}
7
+ export PSI_LOG_REPORT_TO=wandb
8
+
9
+ source /cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/Psi0-yth-v15/.venv-psi/bin/activate
10
+ [ -f .env ] && source .env
11
+
12
+ export WANDB_API_KEY="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"
13
+
14
+ export OMP_NUM_THREADS=8
15
+ export TF_CPP_MIN_LOG_LEVEL=3
16
+
17
+ # Keep JSON sources readable in this script, but pass compact JSON to tyro.
18
+ compact_json() {
19
+ python -c 'import json, sys; print(json.dumps(json.load(sys.stdin), separators=(",", ":")))'
20
+ }
21
+
22
+ PSI_SEED_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/Psi0/dataset/seed_all
23
+ PSI_SEED_METADATA=metadata/seed_metadata_v004.csv
24
+
25
+ PSI_DANCE_MOTION_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/GR00T-WholeBodyControl/gear_sonic_deploy/reference/converted/psi0_dance/gvhmr
26
+ PSI_DANCE_AUDIO_VIDEO_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/dance_1723_video_psi0/dance_1723_video_10s
27
+ PSI_DANCE_VIDEO_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/dance_1723_video_psi0/dance_1723_video_10s
28
+ PSI_DANCE_VIDEO_EMBED_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/dance_1723_video_psi0/dance_1723_video_10s_embeds
29
+ PSI_VIDEO_DEMO_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/videogen/motion_example_0506_demo
30
+ PSI_QWEN3_TTS_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/qwen3-tts
31
+ PSI_VIDEO_CLIP_DATASET_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/videogen/psi0_video_clip_50f_rand8_full
32
+ PSI_DANCE_VIDEO_CLIP_ROOT=$PSI_VIDEO_CLIP_DATASET_ROOT/dance_1723_video_10s
33
+ PSI_MOTION_EXAMPLE_VIDEO_CLIP_ROOT=$PSI_VIDEO_CLIP_DATASET_ROOT/motion_example_0506_demo
34
+ PSI_MOTION_EXAMPLE_VIDEO_EMBED_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/videogen/motion_example_0506_demo_video_embeds
35
+
36
+ if [ -z "$PSI_AUDIO_SOURCES_JSON" ]; then
37
+ PSI_AUDIO_SOURCES_JSON=$(compact_json <<JSON
38
+ [
39
+ {
40
+ "name": "psi0_dance_audio",
41
+ "root_dir": "$PSI_DANCE_MOTION_ROOT",
42
+ "video_dir": "$PSI_DANCE_AUDIO_VIDEO_ROOT",
43
+ "dataset_name": "seed-audio-g1"
44
+ }
45
+ ]
46
+ JSON
47
+ )
48
+ fi
49
+
50
+ if [ -z "$PSI_AUDIO_COMMAND_SOURCES_JSON" ]; then
51
+ PSI_AUDIO_COMMAND_SOURCES_JSON=$(compact_json <<JSON
52
+ [
53
+ {
54
+ "name": "qwen3_tts_seed_audio_command",
55
+ "root_dir": "$PSI_SEED_ROOT",
56
+ "dataset_root": "$PSI_QWEN3_TTS_ROOT",
57
+ "metadata_jsonl": "metadata.jsonl",
58
+ "motion_dir_key": "filename",
59
+ "audio_path_key": "audio_path",
60
+ "instruction_key": "content_short_description",
61
+ "motion_metadata_csv": "$PSI_SEED_METADATA",
62
+ "motion_metadata_key": "filename",
63
+ "motion_metadata_path_key": "move_g1_path",
64
+ "audio_file_extensions": ".wav",
65
+ "audio_task": "command",
66
+ "dataset_name": "seed-audio-command-g1"
67
+ }
68
+ ]
69
+ JSON
70
+ )
71
+ fi
72
+
73
+ if [ -z "$PSI_VIDEO_SOURCES_JSON" ]; then
74
+ PSI_VIDEO_SOURCES_JSON=$(compact_json <<JSON
75
+ [
76
+ {
77
+ "name": "psi0_dance_gvhmr",
78
+ "root_dir": "$PSI_DANCE_MOTION_ROOT",
79
+ "video_dir": "$PSI_DANCE_VIDEO_ROOT",
80
+ "video_embed_dir": "$PSI_DANCE_VIDEO_EMBED_ROOT",
81
+ "dataset_name": "seed-video-g1"
82
+ },
83
+ {
84
+ "name": "motion_example_0506_demo",
85
+ "dataset_root": "$PSI_VIDEO_DEMO_ROOT",
86
+ "metadata_jsonl": "metadata.jsonl",
87
+ "motion_dir_key": "converted_path",
88
+ "video_path_key": "video_path",
89
+ "instruction_key": "video_caption",
90
+ "video_embed_dir": "$PSI_MOTION_EXAMPLE_VIDEO_EMBED_ROOT",
91
+ "dataset_name": "motion-example-video-g1"
92
+ }
93
+ ]
94
+ JSON
95
+ )
96
+ fi
97
+
98
+ if [ -z "$PSI_VIDEO_CLIP_SOURCES_JSON" ]; then
99
+ PSI_VIDEO_CLIP_SOURCES_JSON=$(compact_json <<JSON
100
+ [
101
+ {
102
+ "name": "dance_1723_video_10s_clip50",
103
+ "dataset_root": "$PSI_DANCE_VIDEO_CLIP_ROOT",
104
+ "metadata_jsonl": "metadata.jsonl",
105
+ "motion_dir_key": "converted_path",
106
+ "video_path_key": "video_path",
107
+ "instruction_key": "video_caption",
108
+ "video_embed_dir": "$PSI_DANCE_VIDEO_CLIP_ROOT/embeds",
109
+ "dataset_name": "dance-video-clip-g1"
110
+ },
111
+ {
112
+ "name": "motion_example_0506_demo_clip50",
113
+ "dataset_root": "$PSI_MOTION_EXAMPLE_VIDEO_CLIP_ROOT",
114
+ "metadata_jsonl": "metadata.jsonl",
115
+ "motion_dir_key": "converted_path",
116
+ "video_path_key": "video_path",
117
+ "instruction_key": "video_caption",
118
+ "video_embed_dir": "$PSI_MOTION_EXAMPLE_VIDEO_CLIP_ROOT/embeds",
119
+ "dataset_name": "motion-example-video-clip-g1"
120
+ }
121
+ ]
122
+ JSON
123
+ )
124
+ fi
125
+
126
+ PSI_TEXT_TASK_WEIGHT=0
127
+ PSI_AUDIO_TEXT_TASK_WEIGHT=0.5
128
+ PSI_AUDIO_ONLY_TASK_WEIGHT=0.5
129
+ PSI_AUDIO_COMMAND_TEXT_TASK_WEIGHT=0
130
+ PSI_AUDIO_COMMAND_ONLY_TASK_WEIGHT=0
131
+ PSI_VIDEO_TEXT_TASK_WEIGHT=0
132
+ PSI_VIDEO_ONLY_TASK_WEIGHT=0
133
+ PSI_VIDEO_CLIP_TEXT_TASK_WEIGHT=0
134
+ PSI_VIDEO_CLIP_ONLY_TASK_WEIGHT=0
135
+
136
+ PSI_PRETRAIN_BACKBONE_PATH=/cpfs01/VideoGen/public_ckpt/Qwen/Qwen2.5-Omni-3B
137
+ NUM_PAST_FRAMES=0
138
+ STATE_HORIZON=$((NUM_PAST_FRAMES + 1))
139
+ ACTION_CHUNK_SIZE=30
140
+ ACTION_EXEC_HORIZON=20
141
+ PROGRESS_DIM=5
142
+ VIDEO_SAMPLE_FPS=1.0
143
+ VIDEO_CLIP_SAMPLE_FPS=8.0
144
+ VIDEO_SIZE_PIXELS=50176
145
+ VIDEO_DECODER=ffmpeg
146
+ VIDEO_CACHE_SIZE=64
147
+ DATALOADER_NUM_WORKERS=4
148
+ VAL_DATALOADER_NUM_WORKERS=2
149
+
150
+ NPROC_PER_NODE=$(echo $CUDA_VISIBLE_DEVICES | tr ',' '\n' | wc -l)
151
+
152
+ echo "Training with $NPROC_PER_NODE GPU(s): $CUDA_VISIBLE_DEVICES"
153
+ echo "Text root: $PSI_SEED_ROOT"
154
+ echo "Dance motion root: $PSI_DANCE_MOTION_ROOT"
155
+ echo "Dance audio/video root: $PSI_DANCE_AUDIO_VIDEO_ROOT"
156
+ echo "Dance video root: $PSI_DANCE_VIDEO_ROOT"
157
+ echo "Dance video embed root: $PSI_DANCE_VIDEO_EMBED_ROOT"
158
+ echo "Qwen3-TTS acdudio command root: $PSI_QWEN3_TTS_ROOT"
159
+ echo "Audio sources: $PSI_AUDIO_SOURCES_JSON"
160
+ echo "Audio command sources: $PSI_AUDIO_COMMAND_SOURCES_JSON"
161
+ echo "Video sources: $PSI_VIDEO_SOURCES_JSON"
162
+ echo "Video clip sources: $PSI_VIDEO_CLIP_SOURCES_JSON"
163
+ echo "Task weights: text=$PSI_TEXT_TASK_WEIGHT audio_text=$PSI_AUDIO_TEXT_TASK_WEIGHT audio_only=$PSI_AUDIO_ONLY_TASK_WEIGHT audio_command_text=$PSI_AUDIO_COMMAND_TEXT_TASK_WEIGHT audio_command_only=$PSI_AUDIO_COMMAND_ONLY_TASK_WEIGHT video_text=$PSI_VIDEO_TEXT_TASK_WEIGHT video_only=$PSI_VIDEO_ONLY_TASK_WEIGHT video_clip_text=$PSI_VIDEO_CLIP_TEXT_TASK_WEIGHT video_clip_only=$PSI_VIDEO_CLIP_ONLY_TASK_WEIGHT"
164
+ # echo "Resume run dir: $PSI_PRETRAIN_RUN_DIR"
165
+ echo "TextOp 39D action/state enabled"
166
+ echo "Future progress embedding: enabled, dim $PROGRESS_DIM"
167
+ echo "Backbone path: $PSI_PRETRAIN_BACKBONE_PATH"
168
+ echo "Output dir: $PSI_OUTPUT_DIR"
169
+
170
+ args="posttrain_v1_seed_text_audio_video_mix_psi0_config \
171
+ --seed=292285 \
172
+ --exp=posttrain \
173
+ --timestamp=$(date +"%y%m%d%H%M") \
174
+ --train.name=posttrain_v1_trimodal \
175
+ --train.data_parallel=ddp \
176
+ --train.mixed_precision=bf16 \
177
+ --train.train_batch_size=32 \
178
+ --train.val_batch_size=64 \
179
+ --train.num-workers=$DATALOADER_NUM_WORKERS \
180
+ --train.val-num-workers=$VAL_DATALOADER_NUM_WORKERS \
181
+ --train.resume_from_checkpoint=None \
182
+ --train.max_checkpoints_to_keep=1 \
183
+ --train.gradient_accumulation_steps=1 \
184
+ --train.learning_rate=1e-4 \
185
+ --train.max_training_steps=500000 \
186
+ --train.warmup_ratio=None \
187
+ --train.warmup_steps=1000 \
188
+ --train.checkpointing_steps=1000 \
189
+ --train.validation_steps=1000 \
190
+ --train.val_num_batches=20 \
191
+ --train.max_grad_norm=1.0 \
192
+ --train.lr_scheduler_type=cosine_with_min_lr \
193
+ --train.lr_scheduler_kwargs.betas 0.9 0.999 \
194
+ --train.lr_scheduler_kwargs.weight_decay=0 \
195
+ --train.lr_scheduler_kwargs.eps=1e-8 \
196
+ --log.report_to=$PSI_LOG_REPORT_TO \
197
+ --train.output_dir=$PSI_OUTPUT_DIR \
198
+ --data.text.root-dir=$PSI_SEED_ROOT \
199
+ --data.text.metadata-csv=$PSI_SEED_METADATA \
200
+ --data.text-ratio=$PSI_TEXT_TASK_WEIGHT \
201
+ --data.audio.sources-json=$PSI_AUDIO_SOURCES_JSON \
202
+ --data.audio-ratio=$PSI_AUDIO_TEXT_TASK_WEIGHT \
203
+ --data.audio-only-ratio=$PSI_AUDIO_ONLY_TASK_WEIGHT \
204
+ --data.audio.sample-stride=1 \
205
+ --data.audio.motion-fps=50 \
206
+ --data.audio.audio-sampling-rate=16000 \
207
+ --data.audio.audio-window-seconds=0.6 \
208
+ --data.audio.audio-left-context-seconds=0.0 \
209
+ --data.audio.no-full-audio \
210
+ --data.audio-command.sources-json=$PSI_AUDIO_COMMAND_SOURCES_JSON \
211
+ --data.audio-command-ratio=$PSI_AUDIO_COMMAND_TEXT_TASK_WEIGHT \
212
+ --data.audio-command-only-ratio=$PSI_AUDIO_COMMAND_ONLY_TASK_WEIGHT \
213
+ --data.audio-command.sample-stride=1 \
214
+ --data.audio-command.motion-fps=50 \
215
+ --data.audio-command.audio-sampling-rate=16000 \
216
+ --data.audio-command.audio-window-seconds=1.0 \
217
+ --data.audio-command.audio-left-context-seconds=0.0 \
218
+ --data.audio-command.full-audio \
219
+ --data.video-ratio=$PSI_VIDEO_TEXT_TASK_WEIGHT \
220
+ --data.video-only-ratio=$PSI_VIDEO_ONLY_TASK_WEIGHT \
221
+ --data.video.sample-stride=1 \
222
+ --data.video.motion-fps=50 \
223
+ --data.video.video-sample-fps=$VIDEO_SAMPLE_FPS \
224
+ --data.video.video-size-pixels=$VIDEO_SIZE_PIXELS \
225
+ --data.video.video-decoder=$VIDEO_DECODER \
226
+ --data.video.video-cache-size=$VIDEO_CACHE_SIZE \
227
+ --data.video.sources-json=$PSI_VIDEO_SOURCES_JSON \
228
+ --data.video-clip-text-ratio=$PSI_VIDEO_CLIP_TEXT_TASK_WEIGHT \
229
+ --data.video-clip-only-ratio=$PSI_VIDEO_CLIP_ONLY_TASK_WEIGHT \
230
+ --data.video-clip.sources-json=$PSI_VIDEO_CLIP_SOURCES_JSON \
231
+ --data.video-clip.sample-stride=1 \
232
+ --data.video-clip.motion-fps=50 \
233
+ --data.video-clip.video-sample-fps=$VIDEO_CLIP_SAMPLE_FPS \
234
+ --data.video-clip.video-size-pixels=$VIDEO_SIZE_PIXELS \
235
+ --data.video-clip.video-decoder=$VIDEO_DECODER \
236
+ --data.video-clip.video-cache-size=$VIDEO_CACHE_SIZE \
237
+ --data.video-clip.action-exec-horizon=$ACTION_EXEC_HORIZON \
238
+ --data.sampler=batch_mixture \
239
+ --data.val-task=audio,audio_only \
240
+ --data.transform.repack.num-past-frames=$NUM_PAST_FRAMES \
241
+ --data.transform.repack.textop-actions \
242
+ --data.transform.repack.action-chunk-size=$ACTION_CHUNK_SIZE \
243
+ --data.transform.repack.pad-action-dim=39 \
244
+ --data.transform.repack.pad-state-dim=39 \
245
+ --data.transform.field.action_norm_type=bounds_q99 \
246
+ --data.transform.field.stat-path=assets/stats/seed_all_g1_motion39_textop_stats.json \
247
+ --data.transform.field.normalize-state \
248
+ --data.transform.field.pad-action-dim=39 \
249
+ --data.transform.field.pad-state-dim=39 \
250
+ --data.transform.model.resize.size 240 320 \
251
+ --data.transform.model.center_crop.size 240 320 \
252
+ --data.transform.model.no-img-aug \
253
+ --data.transform.model.text-only \
254
+ --data.transform.model.audio-only \
255
+ --data.transform.model.video-only \
256
+ --data.transform.model.video-size-pixels=$VIDEO_SIZE_PIXELS \
257
+ --model.model_name_or_path=$PSI_PRETRAIN_BACKBONE_PATH \
258
+ --model.noise-scheduler=flow \
259
+ --model.n_conditions=0 \
260
+ --model.action-chunk-size=$ACTION_CHUNK_SIZE \
261
+ --model.action-dim=39 \
262
+ --model.action-exec-horizon=$ACTION_EXEC_HORIZON \
263
+ --model.observation-horizon=$STATE_HORIZON \
264
+ --model.odim=39 \
265
+ --model.view_feature_dim=2048 \
266
+ --model.use-future-progress-embedding \
267
+ --model.future-progress-dim=$PROGRESS_DIM \
268
+ --model.tune-vlm \
269
+ --model.lang-backbone-lr=1e-5 \
270
+ --model.no-use_film \
271
+ --model.no-combined_temb \
272
+ --model.rtc \
273
+ --model.max-delay=11 \
274
+ --train.max-grad-value=1 \
275
+ --train.loss-group-weight.joint-abs=1.0 \
276
+ --train.loss-group-weight.root-delta-xy=3.0 \
277
+ --train.loss-group-weight.root-rot5-delta=10.0 \
278
+ --train.loss-group-weight.root-height=15.0 \
279
+ --train.loss-group-weight.foot-contact=0.1
280
+ "
281
+ torchrun --nproc_per_node=$NPROC_PER_NODE --master_port=29500 scripts/train.py \
282
+ $args "$@"
posttrain-v1-psi0-motion-distill.sh ADDED
@@ -0,0 +1,349 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ pkill -f monitor.py
2
+
3
+ cd /cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/Psi0-yth-v18
4
+ export PYTHONPATH=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/Psi0-yth-v18/src
5
+ export CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7
6
+ export PSI_OUTPUT_DIR=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/Psi0-yth-v18/runs
7
+ export PSI_LOG_REPORT_TO=${PSI_LOG_REPORT_TO:-wandb}
8
+
9
+ source .venv-psi/bin/activate
10
+ [ -f .env ] && source .env
11
+
12
+ export WANDB_API_KEY="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"
13
+
14
+ export OMP_NUM_THREADS=8
15
+ export TF_CPP_MIN_LOG_LEVEL=3
16
+
17
+ compact_json() {
18
+ python -c 'import json, sys; print(json.dumps(json.load(sys.stdin), separators=(",", ":")))'
19
+ }
20
+
21
+ STAGE=${PSI_MOTION_DISTILL_STAGE:-teacher}
22
+ if [ "$1" = "teacher" ] || [ "$1" = "student" ] || [ "$1" = "cache" ]; then
23
+ STAGE=$1
24
+ shift
25
+ fi
26
+
27
+ PSI_SEED_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/Psi0/dataset/seed_all
28
+ PSI_SEED_METADATA=metadata/seed_metadata_v004.csv
29
+
30
+ PSI_DANCE_MOTION_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/GR00T-WholeBodyControl/gear_sonic_deploy/reference/converted/psi0_dance/gvhmr
31
+ PSI_DANCE_AUDIO_VIDEO_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/dance_1723_video_psi0/dance_1723_video_10s
32
+ PSI_DANCE_VIDEO_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/dance_1723_video_psi0/dance_1723_video_10s
33
+ PSI_DANCE_VIDEO_EMBED_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/dance_1723_video_psi0/dance_1723_video_10s_embeds
34
+ PSI_VIDEO_DEMO_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/videogen/motion_example_0506_demo
35
+ PSI_QWEN3_TTS_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/qwen3-tts
36
+ PSI_VIDEO_CLIP_DATASET_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/videogen/psi0_video_clip_50f_rand8_full
37
+ PSI_DANCE_VIDEO_CLIP_ROOT=$PSI_VIDEO_CLIP_DATASET_ROOT/dance_1723_video_10s
38
+ PSI_MOTION_EXAMPLE_VIDEO_CLIP_ROOT=$PSI_VIDEO_CLIP_DATASET_ROOT/motion_example_0506_demo
39
+ PSI_MOTION_EXAMPLE_VIDEO_EMBED_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/videogen/motion_example_0506_demo_video_embeds
40
+
41
+ if [ -z "$PSI_AUDIO_SOURCES_JSON" ]; then
42
+ PSI_AUDIO_SOURCES_JSON=$(compact_json <<JSON
43
+ [
44
+ {
45
+ "name": "psi0_dance_audio",
46
+ "root_dir": "$PSI_DANCE_MOTION_ROOT",
47
+ "video_dir": "$PSI_DANCE_AUDIO_VIDEO_ROOT",
48
+ "dataset_name": "seed-audio-g1"
49
+ }
50
+ ]
51
+ JSON
52
+ )
53
+ fi
54
+
55
+ if [ -z "$PSI_AUDIO_COMMAND_SOURCES_JSON" ]; then
56
+ PSI_AUDIO_COMMAND_SOURCES_JSON=$(compact_json <<JSON
57
+ [
58
+ {
59
+ "name": "qwen3_tts_seed_audio_command",
60
+ "root_dir": "$PSI_SEED_ROOT",
61
+ "dataset_root": "$PSI_QWEN3_TTS_ROOT",
62
+ "metadata_jsonl": "metadata.jsonl",
63
+ "motion_dir_key": "filename",
64
+ "audio_path_key": "audio_path",
65
+ "instruction_key": "content_short_description",
66
+ "motion_metadata_csv": "$PSI_SEED_METADATA",
67
+ "motion_metadata_key": "filename",
68
+ "motion_metadata_path_key": "move_g1_path",
69
+ "audio_file_extensions": ".wav",
70
+ "audio_task": "command",
71
+ "dataset_name": "seed-audio-command-g1"
72
+ }
73
+ ]
74
+ JSON
75
+ )
76
+ fi
77
+
78
+ if [ -z "$PSI_VIDEO_SOURCES_JSON" ]; then
79
+ PSI_VIDEO_SOURCES_JSON=$(compact_json <<JSON
80
+ [
81
+ {
82
+ "name": "psi0_dance_gvhmr",
83
+ "root_dir": "$PSI_DANCE_MOTION_ROOT",
84
+ "video_dir": "$PSI_DANCE_VIDEO_ROOT",
85
+ "video_embed_dir": "$PSI_DANCE_VIDEO_EMBED_ROOT",
86
+ "dataset_name": "seed-video-g1"
87
+ },
88
+ {
89
+ "name": "motion_example_0506_demo",
90
+ "dataset_root": "$PSI_VIDEO_DEMO_ROOT",
91
+ "metadata_jsonl": "metadata.jsonl",
92
+ "motion_dir_key": "converted_path",
93
+ "video_path_key": "video_path",
94
+ "instruction_key": "video_caption",
95
+ "video_embed_dir": "$PSI_MOTION_EXAMPLE_VIDEO_EMBED_ROOT",
96
+ "dataset_name": "motion-example-video-g1"
97
+ }
98
+ ]
99
+ JSON
100
+ )
101
+ fi
102
+
103
+ if [ -z "$PSI_VIDEO_CLIP_SOURCES_JSON" ]; then
104
+ PSI_VIDEO_CLIP_SOURCES_JSON=$(compact_json <<JSON
105
+ [
106
+ {
107
+ "name": "dance_1723_video_10s_clip50",
108
+ "dataset_root": "$PSI_DANCE_VIDEO_CLIP_ROOT",
109
+ "metadata_jsonl": "metadata.jsonl",
110
+ "motion_dir_key": "converted_path",
111
+ "video_path_key": "video_path",
112
+ "instruction_key": "video_caption",
113
+ "video_embed_dir": "$PSI_DANCE_VIDEO_CLIP_ROOT/embeds",
114
+ "dataset_name": "dance-video-clip-g1"
115
+ },
116
+ {
117
+ "name": "motion_example_0506_demo_clip50",
118
+ "dataset_root": "$PSI_MOTION_EXAMPLE_VIDEO_CLIP_ROOT",
119
+ "metadata_jsonl": "metadata.jsonl",
120
+ "motion_dir_key": "converted_path",
121
+ "video_path_key": "video_path",
122
+ "instruction_key": "video_caption",
123
+ "video_embed_dir": "$PSI_MOTION_EXAMPLE_VIDEO_CLIP_ROOT/embeds",
124
+ "dataset_name": "motion-example-video-clip-g1"
125
+ }
126
+ ]
127
+ JSON
128
+ )
129
+ fi
130
+
131
+ PSI_TEXT_TASK_WEIGHT=0
132
+ PSI_AUDIO_TEXT_TASK_WEIGHT=0
133
+ PSI_AUDIO_ONLY_TASK_WEIGHT=0
134
+ PSI_AUDIO_COMMAND_TEXT_TASK_WEIGHT=0
135
+ PSI_AUDIO_COMMAND_ONLY_TASK_WEIGHT=0
136
+ PSI_VIDEO_TEXT_TASK_WEIGHT=0
137
+ PSI_VIDEO_ONLY_TASK_WEIGHT=0
138
+ PSI_VIDEO_CLIP_TEXT_TASK_WEIGHT=0.5
139
+ PSI_VIDEO_CLIP_ONLY_TASK_WEIGHT=0.5
140
+
141
+ PSI_PRETRAIN_BACKBONE_PATH=/cpfs01/VideoGen/public_ckpt/Qwen/Qwen2.5-Omni-3B
142
+ NUM_PAST_FRAMES=0
143
+ STATE_HORIZON=$((NUM_PAST_FRAMES + 1))
144
+ ACTION_CHUNK_SIZE=30
145
+ ACTION_EXEC_HORIZON=20
146
+ MOTION_ENCODER_INPUT_HORIZON=${PSI_MOTION_ENCODER_INPUT_HORIZON:-$ACTION_CHUNK_SIZE}
147
+ PROGRESS_DIM=5
148
+ VIDEO_SAMPLE_FPS=1.0
149
+ VIDEO_CLIP_SAMPLE_FPS=8.0
150
+ VIDEO_SIZE_PIXELS=50176
151
+ VIDEO_DECODER=ffmpeg
152
+ VIDEO_CACHE_SIZE=64
153
+ DATALOADER_NUM_WORKERS=4
154
+ VAL_DATALOADER_NUM_WORKERS=2
155
+ MAX_TRAINING_STEPS=${PSI_MAX_TRAINING_STEPS:-1000000}
156
+ CHECKPOINTING_STEPS=${PSI_CHECKPOINTING_STEPS:-1000}
157
+ VALIDATION_STEPS=${PSI_VALIDATION_STEPS:-1000}
158
+ TEACHER_CHECKPOINT=${PSI_TEACHER_CHECKPOINT:-}
159
+ MOTION_LATENT_NUM_TOKENS=64
160
+ MOTION_LATENT_TOKEN=${PSI_MOTION_LATENT_TOKEN:-"<|motion_latent|>"}
161
+ MOTION_LATENT_TEACHER_ID=psi0_motion_distill
162
+ MOTION_LATENT_CACHE_DIR=${PSI_MOTION_LATENT_CACHE_DIR:-$PSI_VIDEO_CLIP_DATASET_ROOT/motion_latents/$MOTION_LATENT_TEACHER_ID}
163
+ STUDENT_VLM_SCOPE=${PSI_STUDENT_VLM_SCOPE:-video}
164
+ STUDENT_HIDDEN_LOSS_WEIGHT=${PSI_STUDENT_HIDDEN_LOSS_WEIGHT:-1.0}
165
+ STUDENT_HIDDEN_COSINE_LOSS_WEIGHT=${PSI_STUDENT_HIDDEN_COSINE_LOSS_WEIGHT:-0}
166
+ STUDENT_ACTION_LOSS_WEIGHT=${PSI_STUDENT_ACTION_LOSS_WEIGHT:-0}
167
+ TRAIN_BATCH_SIZE=64
168
+ VAL_BATCH_SIZE=64
169
+ if [ "$STAGE" = "student" ]; then
170
+ TRAIN_BATCH_SIZE=32
171
+ VAL_BATCH_SIZE=32
172
+ fi
173
+
174
+ NPROC_PER_NODE=$(echo $CUDA_VISIBLE_DEVICES | tr ',' '\n' | wc -l)
175
+
176
+ if [ "$STAGE" = "student" ] || [ "$STAGE" = "cache" ]; then
177
+ if [ -z "$TEACHER_CHECKPOINT" ]; then
178
+ echo "Set PSI_TEACHER_CHECKPOINT to the stage-1 run dir or checkpoint dir."
179
+ exit 1
180
+ fi
181
+ fi
182
+
183
+ echo "Motion distill stage: $STAGE"
184
+ echo "Training with $NPROC_PER_NODE GPU(s): $CUDA_VISIBLE_DEVICES"
185
+ echo "Video clip sources: $PSI_VIDEO_CLIP_SOURCES_JSON"
186
+ echo "Motion encoder input horizon: $MOTION_ENCODER_INPUT_HORIZON"
187
+ echo "Teacher checkpoint: $TEACHER_CHECKPOINT"
188
+ echo "Motion latent cache dir: $MOTION_LATENT_CACHE_DIR"
189
+ echo "Motion latent tokens: $MOTION_LATENT_NUM_TOKENS x $MOTION_LATENT_TOKEN"
190
+ echo "Student VLM scope: $STUDENT_VLM_SCOPE"
191
+ echo "Student hidden loss weight: $STUDENT_HIDDEN_LOSS_WEIGHT"
192
+ echo "Student hidden cosine loss weight: $STUDENT_HIDDEN_COSINE_LOSS_WEIGHT"
193
+ echo "Student action loss weight: $STUDENT_ACTION_LOSS_WEIGHT"
194
+ echo "Per-GPU batch size: train=$TRAIN_BATCH_SIZE val=$VAL_BATCH_SIZE"
195
+ echo "Backbone path: $PSI_PRETRAIN_BACKBONE_PATH"
196
+ echo "Output dir: $PSI_OUTPUT_DIR"
197
+
198
+ CONFIG_STAGE=$STAGE
199
+ if [ "$STAGE" = "cache" ]; then
200
+ CONFIG_STAGE=teacher
201
+ fi
202
+
203
+ stage_args="--model.motion-distill-stage=$CONFIG_STAGE \
204
+ --model.motion-encoder-input-horizon=$MOTION_ENCODER_INPUT_HORIZON \
205
+ --model.motion-latent-num-tokens=$MOTION_LATENT_NUM_TOKENS \
206
+ --model.motion-latent-token=$MOTION_LATENT_TOKEN \
207
+ --model.motion-encoder-hidden-dim=2048"
208
+
209
+ if [ "$STAGE" = "teacher" ]; then
210
+ stage_args="$stage_args --model.no-tune-vlm"
211
+ elif [ "$STAGE" = "student" ]; then
212
+ stage_args="$stage_args \
213
+ --model.tune-vlm \
214
+ --model.motion-teacher-checkpoint=$TEACHER_CHECKPOINT \
215
+ --model.motion-latent-cache-dir=$MOTION_LATENT_CACHE_DIR \
216
+ --model.motion-distill-vlm-train-scope=$STUDENT_VLM_SCOPE \
217
+ --model.motion-hidden-loss-weight=$STUDENT_HIDDEN_LOSS_WEIGHT \
218
+ --model.motion-hidden-cosine-loss-weight=$STUDENT_HIDDEN_COSINE_LOSS_WEIGHT \
219
+ --model.motion-student-action-loss-weight=$STUDENT_ACTION_LOSS_WEIGHT \
220
+ --model.motion-latent-cache-strict"
221
+ elif [ "$STAGE" = "cache" ]; then
222
+ stage_args="$stage_args \
223
+ --model.no-tune-vlm \
224
+ --model.motion-teacher-checkpoint=$TEACHER_CHECKPOINT \
225
+ --model.motion-latent-cache-dir=$MOTION_LATENT_CACHE_DIR"
226
+ else
227
+ echo "Unknown stage: $STAGE"
228
+ exit 1
229
+ fi
230
+
231
+ args="posttrain_v1_seed_text_audio_video_mix_psi0_motion_distill_config \
232
+ --seed=292285 \
233
+ --exp=posttrain \
234
+ --timestamp=$(date +"%y%m%d%H%M") \
235
+ --train.name=posttrain_v1_motion_distill \
236
+ --train.data_parallel=ddp \
237
+ --train.mixed_precision=bf16 \
238
+ --train.train_batch_size=$TRAIN_BATCH_SIZE \
239
+ --train.val_batch_size=$VAL_BATCH_SIZE \
240
+ --train.num-workers=$DATALOADER_NUM_WORKERS \
241
+ --train.val-num-workers=$VAL_DATALOADER_NUM_WORKERS \
242
+ --train.resume_from_checkpoint=None \
243
+ --train.max_checkpoints_to_keep=1 \
244
+ --train.gradient_accumulation_steps=1 \
245
+ --train.learning_rate=1e-4 \
246
+ --train.max_training_steps=$MAX_TRAINING_STEPS \
247
+ --train.warmup_ratio=None \
248
+ --train.warmup_steps=1000 \
249
+ --train.checkpointing_steps=$CHECKPOINTING_STEPS \
250
+ --train.validation_steps=$VALIDATION_STEPS \
251
+ --train.val_num_batches=20 \
252
+ --train.max_grad_norm=1.0 \
253
+ --train.lr_scheduler_type=cosine_with_min_lr \
254
+ --train.lr_scheduler_kwargs.betas 0.9 0.999 \
255
+ --train.lr_scheduler_kwargs.weight_decay=0 \
256
+ --train.lr_scheduler_kwargs.eps=1e-8 \
257
+ --log.report_to=$PSI_LOG_REPORT_TO \
258
+ --train.output_dir=$PSI_OUTPUT_DIR \
259
+ --data.text.root-dir=$PSI_SEED_ROOT \
260
+ --data.text.metadata-csv=$PSI_SEED_METADATA \
261
+ --data.text-ratio=$PSI_TEXT_TASK_WEIGHT \
262
+ --data.audio.sources-json=$PSI_AUDIO_SOURCES_JSON \
263
+ --data.audio-ratio=$PSI_AUDIO_TEXT_TASK_WEIGHT \
264
+ --data.audio-only-ratio=$PSI_AUDIO_ONLY_TASK_WEIGHT \
265
+ --data.audio.sample-stride=1 \
266
+ --data.audio.motion-fps=50 \
267
+ --data.audio.audio-sampling-rate=16000 \
268
+ --data.audio.audio-window-seconds=1.0 \
269
+ --data.audio.audio-left-context-seconds=0.0 \
270
+ --data.audio.full-audio \
271
+ --data.audio-command.sources-json=$PSI_AUDIO_COMMAND_SOURCES_JSON \
272
+ --data.audio-command-ratio=$PSI_AUDIO_COMMAND_TEXT_TASK_WEIGHT \
273
+ --data.audio-command-only-ratio=$PSI_AUDIO_COMMAND_ONLY_TASK_WEIGHT \
274
+ --data.audio-command.sample-stride=1 \
275
+ --data.audio-command.motion-fps=50 \
276
+ --data.audio-command.audio-sampling-rate=16000 \
277
+ --data.audio-command.audio-window-seconds=1.0 \
278
+ --data.audio-command.audio-left-context-seconds=0.0 \
279
+ --data.audio-command.full-audio \
280
+ --data.video-ratio=$PSI_VIDEO_TEXT_TASK_WEIGHT \
281
+ --data.video-only-ratio=$PSI_VIDEO_ONLY_TASK_WEIGHT \
282
+ --data.video.sample-stride=1 \
283
+ --data.video.motion-fps=50 \
284
+ --data.video.video-sample-fps=$VIDEO_SAMPLE_FPS \
285
+ --data.video.video-size-pixels=$VIDEO_SIZE_PIXELS \
286
+ --data.video.video-decoder=$VIDEO_DECODER \
287
+ --data.video.video-cache-size=$VIDEO_CACHE_SIZE \
288
+ --data.video.sources-json=$PSI_VIDEO_SOURCES_JSON \
289
+ --data.video-clip-text-ratio=$PSI_VIDEO_CLIP_TEXT_TASK_WEIGHT \
290
+ --data.video-clip-only-ratio=$PSI_VIDEO_CLIP_ONLY_TASK_WEIGHT \
291
+ --data.video-clip.sources-json=$PSI_VIDEO_CLIP_SOURCES_JSON \
292
+ --data.video-clip.sample-stride=1 \
293
+ --data.video-clip.motion-fps=50 \
294
+ --data.video-clip.video-sample-fps=$VIDEO_CLIP_SAMPLE_FPS \
295
+ --data.video-clip.video-size-pixels=$VIDEO_SIZE_PIXELS \
296
+ --data.video-clip.video-decoder=$VIDEO_DECODER \
297
+ --data.video-clip.video-cache-size=$VIDEO_CACHE_SIZE \
298
+ --data.video-clip.action-exec-horizon=$ACTION_EXEC_HORIZON \
299
+ --data.sampler=batch_mixture \
300
+ --data.val-task=video_clip,video_clip_only \
301
+ --data.transform.repack.num-past-frames=$NUM_PAST_FRAMES \
302
+ --data.transform.repack.textop-actions \
303
+ --data.transform.repack.action-chunk-size=$ACTION_CHUNK_SIZE \
304
+ --data.transform.repack.pad-action-dim=39 \
305
+ --data.transform.repack.pad-state-dim=39 \
306
+ --data.transform.field.action_norm_type=bounds_q99 \
307
+ --data.transform.field.stat-path=assets/stats/seed_all_g1_motion39_textop_stats.json \
308
+ --data.transform.field.normalize-state \
309
+ --data.transform.field.pad-action-dim=39 \
310
+ --data.transform.field.pad-state-dim=39 \
311
+ --data.transform.model.resize.size 240 320 \
312
+ --data.transform.model.center_crop.size 240 320 \
313
+ --data.transform.model.no-img-aug \
314
+ --data.transform.model.text-only \
315
+ --data.transform.model.audio-only \
316
+ --data.transform.model.video-only \
317
+ --data.transform.model.video-size-pixels=$VIDEO_SIZE_PIXELS \
318
+ --data.transform.model.motion-latent-token=$MOTION_LATENT_TOKEN \
319
+ --data.transform.model.motion-latent-num-tokens=$MOTION_LATENT_NUM_TOKENS \
320
+ --model.model_name_or_path=$PSI_PRETRAIN_BACKBONE_PATH \
321
+ --model.noise-scheduler=flow \
322
+ --model.n_conditions=0 \
323
+ --model.action-chunk-size=$ACTION_CHUNK_SIZE \
324
+ --model.action-dim=39 \
325
+ --model.action-exec-horizon=$ACTION_EXEC_HORIZON \
326
+ --model.observation-horizon=$STATE_HORIZON \
327
+ --model.odim=39 \
328
+ --model.view_feature_dim=2048 \
329
+ --model.use-future-progress-embedding \
330
+ --model.future-progress-dim=$PROGRESS_DIM \
331
+ --model.no-use_film \
332
+ --model.no-combined_temb \
333
+ --model.rtc \
334
+ --model.max-delay=11 \
335
+ --train.max-grad-value=1 \
336
+ --train.loss-group-weight.joint-abs=1.0 \
337
+ --train.loss-group-weight.root-delta-xy=1.0 \
338
+ --train.loss-group-weight.root-rot5-delta=1.0 \
339
+ --train.loss-group-weight.root-height=1.0 \
340
+ --train.loss-group-weight.foot-contact=1.0 \
341
+ $stage_args
342
+ "
343
+
344
+ if [ "$STAGE" = "cache" ]; then
345
+ python scripts/save_motion_teacher_hidden_cache.py $args "$@"
346
+ else
347
+ torchrun --nproc_per_node=$NPROC_PER_NODE --master_port=${MASTER_PORT:-29510} scripts/train.py \
348
+ $args "$@"
349
+ fi
posttrain-v1-psi0-text-ckpt-continue-video-clip-tune-vlm-direct-dense.sh ADDED
@@ -0,0 +1,305 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ pkill -f monitor.py
2
+
3
+ cd /cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/Psi0-yth-v18
4
+ export PYTHONPATH=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/Psi0-yth-v18/src
5
+ export CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7
6
+ export PSI_OUTPUT_DIR=${PSI_OUTPUT_DIR:-/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/Psi0-yth-v18/runs}
7
+ export PSI_LOG_REPORT_TO=wandb
8
+
9
+ source /cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/Psi0-yth-v15/.venv-psi/bin/activate
10
+ [ -f .env ] && source .env
11
+
12
+ export WANDB_API_KEY="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"
13
+
14
+ export OMP_NUM_THREADS=8
15
+ export TF_CPP_MIN_LOG_LEVEL=3
16
+
17
+ # Keep JSON sources readable in this script, but pass compact JSON to tyro.
18
+ compact_json() {
19
+ python -c 'import json, sys; print(json.dumps(json.load(sys.stdin), separators=(",", ":")))'
20
+ }
21
+
22
+ PSI_SEED_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/Psi0/dataset/seed_all
23
+ PSI_SEED_METADATA=metadata/seed_metadata_v004.csv
24
+
25
+ PSI_DANCE_MOTION_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/GR00T-WholeBodyControl/gear_sonic_deploy/reference/converted/psi0_dance/gvhmr
26
+ PSI_DANCE_AUDIO_VIDEO_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/dance_1723_video_psi0/dance_1723_video_10s
27
+ PSI_DANCE_VIDEO_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/dance_1723_video_psi0/dance_1723_video_10s
28
+ PSI_DANCE_VIDEO_EMBED_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/dance_1723_video_psi0/dance_1723_video_10s_embeds
29
+ PSI_VIDEO_DEMO_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/videogen/motion_example_0506_demo
30
+ PSI_QWEN3_TTS_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/qwen3-tts
31
+ PSI_VIDEO_CLIP_DATASET_ROOT=${PSI_VIDEO_CLIP_DATASET_ROOT:-/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/videogen/psi0_video_clip_50f_dense_stride1_direct_full}
32
+ PSI_DANCE_VIDEO_CLIP_ROOT=$PSI_VIDEO_CLIP_DATASET_ROOT/dance_1723_video_10s
33
+ PSI_MOTION_EXAMPLE_VIDEO_CLIP_ROOT=$PSI_VIDEO_CLIP_DATASET_ROOT/motion_example_0506_demo
34
+ PSI_DANCE_VIDEO_CLIP_EMBED_ROOT=${PSI_DANCE_VIDEO_CLIP_EMBED_ROOT:-$PSI_DANCE_VIDEO_CLIP_ROOT/embeds}
35
+ PSI_MOTION_EXAMPLE_VIDEO_CLIP_EMBED_ROOT=${PSI_MOTION_EXAMPLE_VIDEO_CLIP_EMBED_ROOT:-$PSI_MOTION_EXAMPLE_VIDEO_CLIP_ROOT/embeds}
36
+ PSI_MOTION_EXAMPLE_VIDEO_EMBED_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/videogen/motion_example_0506_demo_video_embeds
37
+
38
+ if [ -z "$PSI_AUDIO_SOURCES_JSON" ]; then
39
+ PSI_AUDIO_SOURCES_JSON=$(compact_json <<JSON
40
+ [
41
+ {
42
+ "name": "psi0_dance_audio",
43
+ "root_dir": "$PSI_DANCE_MOTION_ROOT",
44
+ "video_dir": "$PSI_DANCE_AUDIO_VIDEO_ROOT",
45
+ "dataset_name": "seed-audio-g1"
46
+ }
47
+ ]
48
+ JSON
49
+ )
50
+ fi
51
+
52
+ if [ -z "$PSI_AUDIO_COMMAND_SOURCES_JSON" ]; then
53
+ PSI_AUDIO_COMMAND_SOURCES_JSON=$(compact_json <<JSON
54
+ [
55
+ {
56
+ "name": "qwen3_tts_seed_audio_command",
57
+ "root_dir": "$PSI_SEED_ROOT",
58
+ "dataset_root": "$PSI_QWEN3_TTS_ROOT",
59
+ "metadata_jsonl": "metadata.jsonl",
60
+ "motion_dir_key": "filename",
61
+ "audio_path_key": "audio_path",
62
+ "instruction_key": "content_short_description",
63
+ "motion_metadata_csv": "$PSI_SEED_METADATA",
64
+ "motion_metadata_key": "filename",
65
+ "motion_metadata_path_key": "move_g1_path",
66
+ "audio_file_extensions": ".wav",
67
+ "audio_task": "command",
68
+ "dataset_name": "seed-audio-command-g1"
69
+ }
70
+ ]
71
+ JSON
72
+ )
73
+ fi
74
+
75
+ if [ -z "$PSI_VIDEO_SOURCES_JSON" ]; then
76
+ PSI_VIDEO_SOURCES_JSON=$(compact_json <<JSON
77
+ [
78
+ {
79
+ "name": "psi0_dance_gvhmr",
80
+ "root_dir": "$PSI_DANCE_MOTION_ROOT",
81
+ "video_dir": "$PSI_DANCE_VIDEO_ROOT",
82
+ "video_embed_dir": "$PSI_DANCE_VIDEO_EMBED_ROOT",
83
+ "dataset_name": "seed-video-g1"
84
+ },
85
+ {
86
+ "name": "motion_example_0506_demo",
87
+ "dataset_root": "$PSI_VIDEO_DEMO_ROOT",
88
+ "metadata_jsonl": "metadata.jsonl",
89
+ "motion_dir_key": "converted_path",
90
+ "video_path_key": "video_path",
91
+ "instruction_key": "video_caption",
92
+ "video_embed_dir": "$PSI_MOTION_EXAMPLE_VIDEO_EMBED_ROOT",
93
+ "dataset_name": "motion-example-video-g1"
94
+ }
95
+ ]
96
+ JSON
97
+ )
98
+ fi
99
+
100
+ if [ -z "$PSI_VIDEO_CLIP_SOURCES_JSON" ]; then
101
+ PSI_VIDEO_CLIP_SOURCES_JSON=$(compact_json <<JSON
102
+ [
103
+ {
104
+ "name": "dance_1723_video_10s_clip50",
105
+ "dataset_root": "$PSI_DANCE_VIDEO_CLIP_ROOT",
106
+ "metadata_jsonl": "metadata.jsonl",
107
+ "motion_dir_key": "converted_path",
108
+ "video_path_key": "video_path",
109
+ "instruction_key": "video_caption",
110
+ "video_embed_dir": "$PSI_DANCE_VIDEO_CLIP_EMBED_ROOT",
111
+ "dataset_name": "dance-video-clip-g1"
112
+ },
113
+ {
114
+ "name": "motion_example_0506_demo_clip50",
115
+ "dataset_root": "$PSI_MOTION_EXAMPLE_VIDEO_CLIP_ROOT",
116
+ "metadata_jsonl": "metadata.jsonl",
117
+ "motion_dir_key": "converted_path",
118
+ "video_path_key": "video_path",
119
+ "instruction_key": "video_caption",
120
+ "video_embed_dir": "$PSI_MOTION_EXAMPLE_VIDEO_CLIP_EMBED_ROOT",
121
+ "dataset_name": "motion-example-video-clip-g1"
122
+ }
123
+ ]
124
+ JSON
125
+ )
126
+ fi
127
+
128
+ PSI_RESUME_TEXT_CHECKPOINT=${PSI_RESUME_TEXT_CHECKPOINT:-/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/Psi0-yth-v15/runs/posttrain_v1_trimodal/posttrain.sg.flow1000.cwml.lr1.0e-04.b256.gpus8.2605310619/checkpoints/ckpt_280000}
129
+
130
+ PSI_TEXT_TASK_WEIGHT=0.5
131
+ PSI_AUDIO_TEXT_TASK_WEIGHT=0
132
+ PSI_AUDIO_ONLY_TASK_WEIGHT=0
133
+ PSI_AUDIO_COMMAND_TEXT_TASK_WEIGHT=0
134
+ PSI_AUDIO_COMMAND_ONLY_TASK_WEIGHT=0
135
+ PSI_VIDEO_TEXT_TASK_WEIGHT=0
136
+ PSI_VIDEO_ONLY_TASK_WEIGHT=0
137
+ PSI_VIDEO_CLIP_TEXT_TASK_WEIGHT=0.25
138
+ PSI_VIDEO_CLIP_ONLY_TASK_WEIGHT=0.25
139
+
140
+ PSI_PRETRAIN_BACKBONE_PATH=/cpfs01/VideoGen/public_ckpt/Qwen/Qwen2.5-Omni-3B
141
+ NUM_PAST_FRAMES=0
142
+ STATE_HORIZON=$((NUM_PAST_FRAMES + 1))
143
+ ACTION_CHUNK_SIZE=30
144
+ ACTION_EXEC_HORIZON=20
145
+ PROGRESS_DIM=5
146
+ VIDEO_SAMPLE_FPS=1.0
147
+ VIDEO_CLIP_SAMPLE_FPS=8.0
148
+ VIDEO_SIZE_PIXELS=50176
149
+ VIDEO_DECODER=ffmpeg
150
+ VIDEO_CACHE_SIZE=64
151
+ VIDEO_CLIP_MOTION_CACHE_SIZE=256
152
+ VIDEO_CLIP_TRUST_METADATA=${VIDEO_CLIP_TRUST_METADATA:-1}
153
+ VIDEO_CLIP_RECORDS_CACHE_DIR=${VIDEO_CLIP_RECORDS_CACHE_DIR:-$PSI_VIDEO_CLIP_DATASET_ROOT/.index_cache}
154
+ if [ "$VIDEO_CLIP_TRUST_METADATA" = "1" ] || [ "$VIDEO_CLIP_TRUST_METADATA" = "true" ] || [ "$VIDEO_CLIP_TRUST_METADATA" = "True" ]; then
155
+ VIDEO_CLIP_TRUST_METADATA_FLAG="--data.video-clip.trust-metadata"
156
+ else
157
+ VIDEO_CLIP_TRUST_METADATA_FLAG="--data.video-clip.no-trust-metadata"
158
+ fi
159
+ DATALOADER_NUM_WORKERS=4
160
+ VAL_DATALOADER_NUM_WORKERS=2
161
+
162
+ if [ ! -f "$PSI_RESUME_TEXT_CHECKPOINT/model.safetensors" ] || [ ! -f "$PSI_RESUME_TEXT_CHECKPOINT/optimizer.bin" ] || [ ! -f "$PSI_RESUME_TEXT_CHECKPOINT/scheduler.bin" ]; then
163
+ echo "ERROR: resume checkpoint is not a full training-state checkpoint: $PSI_RESUME_TEXT_CHECKPOINT" >&2
164
+ exit 1
165
+ fi
166
+
167
+ NPROC_PER_NODE=$(echo $CUDA_VISIBLE_DEVICES | tr ',' '\n' | wc -l)
168
+
169
+ echo "Training with $NPROC_PER_NODE GPU(s): $CUDA_VISIBLE_DEVICES"
170
+ echo "Text root: $PSI_SEED_ROOT"
171
+ echo "Dance motion root: $PSI_DANCE_MOTION_ROOT"
172
+ echo "Dance audio/video root: $PSI_DANCE_AUDIO_VIDEO_ROOT"
173
+ echo "Dance video root: $PSI_DANCE_VIDEO_ROOT"
174
+ echo "Dance video embed root: $PSI_DANCE_VIDEO_EMBED_ROOT"
175
+ echo "Qwen3-TTS acdudio command root: $PSI_QWEN3_TTS_ROOT"
176
+ echo "Audio sources: $PSI_AUDIO_SOURCES_JSON"
177
+ echo "Audio command sources: $PSI_AUDIO_COMMAND_SOURCES_JSON"
178
+ echo "Video sources: $PSI_VIDEO_SOURCES_JSON"
179
+ echo "Video clip sources: $PSI_VIDEO_CLIP_SOURCES_JSON"
180
+ echo "Task weights: text=$PSI_TEXT_TASK_WEIGHT audio_text=$PSI_AUDIO_TEXT_TASK_WEIGHT audio_only=$PSI_AUDIO_ONLY_TASK_WEIGHT audio_command_text=$PSI_AUDIO_COMMAND_TEXT_TASK_WEIGHT audio_command_only=$PSI_AUDIO_COMMAND_ONLY_TASK_WEIGHT video_text=$PSI_VIDEO_TEXT_TASK_WEIGHT video_only=$PSI_VIDEO_ONLY_TASK_WEIGHT video_clip_text=$PSI_VIDEO_CLIP_TEXT_TASK_WEIGHT video_clip_only=$PSI_VIDEO_CLIP_ONLY_TASK_WEIGHT"
181
+ echo "Resume full training state from: $PSI_RESUME_TEXT_CHECKPOINT"
182
+ echo "TextOp 39D action/state enabled"
183
+ echo "Future progress embedding: enabled, dim $PROGRESS_DIM"
184
+ echo "Backbone path: $PSI_PRETRAIN_BACKBONE_PATH"
185
+ echo "Output dir: $PSI_OUTPUT_DIR"
186
+ echo "Video clip trust metadata: $VIDEO_CLIP_TRUST_METADATA"
187
+ echo "Video clip records cache dir: $VIDEO_CLIP_RECORDS_CACHE_DIR"
188
+
189
+ args="posttrain_v1_seed_text_audio_video_mix_psi0_config \
190
+ --seed=292285 \
191
+ --exp=posttrain \
192
+ --timestamp=$(date +"%y%m%d%H%M") \
193
+ --train.name=posttrain_v1_trimodal \
194
+ --train.data_parallel=ddp \
195
+ --train.mixed_precision=bf16 \
196
+ --train.train_batch_size=32 \
197
+ --train.val_batch_size=64 \
198
+ --train.num-workers=$DATALOADER_NUM_WORKERS \
199
+ --train.val-num-workers=$VAL_DATALOADER_NUM_WORKERS \
200
+ --train.resume_from_checkpoint=$PSI_RESUME_TEXT_CHECKPOINT \
201
+ --train.resume-load-full-state \
202
+ --train.max_checkpoints_to_keep=1 \
203
+ --train.gradient_accumulation_steps=1 \
204
+ --train.learning_rate=1e-4 \
205
+ --train.max_training_steps=800000 \
206
+ --train.warmup_ratio=None \
207
+ --train.warmup_steps=1000 \
208
+ --train.checkpointing_steps=1000 \
209
+ --train.validation_steps=1000 \
210
+ --train.val_num_batches=20 \
211
+ --train.max_grad_norm=1.0 \
212
+ --train.lr_scheduler_type=cosine_with_min_lr \
213
+ --train.lr_scheduler_kwargs.betas 0.9 0.999 \
214
+ --train.lr_scheduler_kwargs.weight_decay=0 \
215
+ --train.lr_scheduler_kwargs.eps=1e-8 \
216
+ --log.report_to=$PSI_LOG_REPORT_TO \
217
+ --train.output_dir=$PSI_OUTPUT_DIR \
218
+ --data.text.root-dir=$PSI_SEED_ROOT \
219
+ --data.text.metadata-csv=$PSI_SEED_METADATA \
220
+ --data.text-ratio=$PSI_TEXT_TASK_WEIGHT \
221
+ --data.audio.sources-json=$PSI_AUDIO_SOURCES_JSON \
222
+ --data.audio-ratio=$PSI_AUDIO_TEXT_TASK_WEIGHT \
223
+ --data.audio-only-ratio=$PSI_AUDIO_ONLY_TASK_WEIGHT \
224
+ --data.audio.sample-stride=1 \
225
+ --data.audio.motion-fps=50 \
226
+ --data.audio.audio-sampling-rate=16000 \
227
+ --data.audio.audio-window-seconds=1.0 \
228
+ --data.audio.audio-left-context-seconds=0.0 \
229
+ --data.audio.full-audio \
230
+ --data.audio-command.sources-json=$PSI_AUDIO_COMMAND_SOURCES_JSON \
231
+ --data.audio-command-ratio=$PSI_AUDIO_COMMAND_TEXT_TASK_WEIGHT \
232
+ --data.audio-command-only-ratio=$PSI_AUDIO_COMMAND_ONLY_TASK_WEIGHT \
233
+ --data.audio-command.sample-stride=1 \
234
+ --data.audio-command.motion-fps=50 \
235
+ --data.audio-command.audio-sampling-rate=16000 \
236
+ --data.audio-command.audio-window-seconds=1.0 \
237
+ --data.audio-command.audio-left-context-seconds=0.0 \
238
+ --data.audio-command.full-audio \
239
+ --data.video-ratio=$PSI_VIDEO_TEXT_TASK_WEIGHT \
240
+ --data.video-only-ratio=$PSI_VIDEO_ONLY_TASK_WEIGHT \
241
+ --data.video.sample-stride=1 \
242
+ --data.video.motion-fps=50 \
243
+ --data.video.video-sample-fps=$VIDEO_SAMPLE_FPS \
244
+ --data.video.video-size-pixels=$VIDEO_SIZE_PIXELS \
245
+ --data.video.video-decoder=$VIDEO_DECODER \
246
+ --data.video.video-cache-size=$VIDEO_CACHE_SIZE \
247
+ --data.video.sources-json=$PSI_VIDEO_SOURCES_JSON \
248
+ --data.video-clip-text-ratio=$PSI_VIDEO_CLIP_TEXT_TASK_WEIGHT \
249
+ --data.video-clip-only-ratio=$PSI_VIDEO_CLIP_ONLY_TASK_WEIGHT \
250
+ --data.video-clip.sources-json=$PSI_VIDEO_CLIP_SOURCES_JSON \
251
+ --data.video-clip.sample-stride=1 \
252
+ --data.video-clip.motion-fps=50 \
253
+ --data.video-clip.video-sample-fps=$VIDEO_CLIP_SAMPLE_FPS \
254
+ --data.video-clip.video-size-pixels=$VIDEO_SIZE_PIXELS \
255
+ --data.video-clip.video-decoder=$VIDEO_DECODER \
256
+ --data.video-clip.video-cache-size=$VIDEO_CACHE_SIZE \
257
+ --data.video-clip.motion-cache-size=$VIDEO_CLIP_MOTION_CACHE_SIZE \
258
+ $VIDEO_CLIP_TRUST_METADATA_FLAG \
259
+ --data.video-clip.records-cache-dir=$VIDEO_CLIP_RECORDS_CACHE_DIR \
260
+ --data.video-clip.action-exec-horizon=$ACTION_EXEC_HORIZON \
261
+ --data.sampler=batch_mixture \
262
+ --data.val-task=text,video_clip,video_clip_only \
263
+ --data.transform.repack.num-past-frames=$NUM_PAST_FRAMES \
264
+ --data.transform.repack.textop-actions \
265
+ --data.transform.repack.action-chunk-size=$ACTION_CHUNK_SIZE \
266
+ --data.transform.repack.pad-action-dim=39 \
267
+ --data.transform.repack.pad-state-dim=39 \
268
+ --data.transform.field.action_norm_type=bounds_q99 \
269
+ --data.transform.field.stat-path=assets/stats/seed_all_g1_motion39_textop_stats.json \
270
+ --data.transform.field.normalize-state \
271
+ --data.transform.field.pad-action-dim=39 \
272
+ --data.transform.field.pad-state-dim=39 \
273
+ --data.transform.model.resize.size 240 320 \
274
+ --data.transform.model.center_crop.size 240 320 \
275
+ --data.transform.model.no-img-aug \
276
+ --data.transform.model.text-only \
277
+ --data.transform.model.audio-only \
278
+ --data.transform.model.video-only \
279
+ --data.transform.model.video-size-pixels=$VIDEO_SIZE_PIXELS \
280
+ --model.model_name_or_path=$PSI_PRETRAIN_BACKBONE_PATH \
281
+ --model.noise-scheduler=flow \
282
+ --model.n_conditions=0 \
283
+ --model.action-chunk-size=$ACTION_CHUNK_SIZE \
284
+ --model.action-dim=39 \
285
+ --model.action-exec-horizon=$ACTION_EXEC_HORIZON \
286
+ --model.observation-horizon=$STATE_HORIZON \
287
+ --model.odim=39 \
288
+ --model.view_feature_dim=2048 \
289
+ --model.use-future-progress-embedding \
290
+ --model.future-progress-dim=$PROGRESS_DIM \
291
+ --model.tune-vlm \
292
+ --model.lang-backbone-lr=1e-5 \
293
+ --model.no-use_film \
294
+ --model.no-combined_temb \
295
+ --model.rtc \
296
+ --model.max-delay=11 \
297
+ --train.max-grad-value=1 \
298
+ --train.loss-group-weight.joint-abs=1.0 \
299
+ --train.loss-group-weight.root-delta-xy=3.0 \
300
+ --train.loss-group-weight.root-rot5-delta=10.0 \
301
+ --train.loss-group-weight.root-height=15.0 \
302
+ --train.loss-group-weight.foot-contact=0.1
303
+ "
304
+ torchrun --nproc_per_node=$NPROC_PER_NODE --master_port=29500 scripts/train.py \
305
+ $args "$@"
posttrain-v1-psi0-text-ckpt-continue-video-clip-tune-vlm.sh ADDED
@@ -0,0 +1,290 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ pkill -f monitor.py
2
+
3
+ cd /cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/Psi0-yth-v18
4
+ export PYTHONPATH=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/Psi0-yth-v18/src
5
+ export CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7
6
+ export PSI_OUTPUT_DIR=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/Psi0-yth-v18/runs
7
+ export PSI_LOG_REPORT_TO=wandb
8
+
9
+ source .venv-psi/bin/activate
10
+ [ -f .env ] && source .env
11
+
12
+ export WANDB_API_KEY="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"
13
+
14
+ export OMP_NUM_THREADS=8
15
+ export TF_CPP_MIN_LOG_LEVEL=3
16
+
17
+ # Keep JSON sources readable in this script, but pass compact JSON to tyro.
18
+ compact_json() {
19
+ python -c 'import json, sys; print(json.dumps(json.load(sys.stdin), separators=(",", ":")))'
20
+ }
21
+
22
+ PSI_SEED_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/Psi0/dataset/seed_all
23
+ PSI_SEED_METADATA=metadata/seed_metadata_v004.csv
24
+
25
+ PSI_DANCE_MOTION_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/GR00T-WholeBodyControl/gear_sonic_deploy/reference/converted/psi0_dance/gvhmr
26
+ PSI_DANCE_AUDIO_VIDEO_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/dance_1723_video_psi0/dance_1723_video_10s
27
+ PSI_DANCE_VIDEO_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/dance_1723_video_psi0/dance_1723_video_10s
28
+ PSI_DANCE_VIDEO_EMBED_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/dance_1723_video_psi0/dance_1723_video_10s_embeds
29
+ PSI_VIDEO_DEMO_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/videogen/motion_example_0506_demo
30
+ PSI_QWEN3_TTS_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/qwen3-tts
31
+ PSI_VIDEO_CLIP_DATASET_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/videogen/psi0_video_clip_50f_rand8_full
32
+ PSI_DANCE_VIDEO_CLIP_ROOT=$PSI_VIDEO_CLIP_DATASET_ROOT/dance_1723_video_10s
33
+ PSI_MOTION_EXAMPLE_VIDEO_CLIP_ROOT=$PSI_VIDEO_CLIP_DATASET_ROOT/motion_example_0506_demo
34
+ PSI_MOTION_EXAMPLE_VIDEO_EMBED_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/videogen/motion_example_0506_demo_video_embeds
35
+
36
+ if [ -z "$PSI_AUDIO_SOURCES_JSON" ]; then
37
+ PSI_AUDIO_SOURCES_JSON=$(compact_json <<JSON
38
+ [
39
+ {
40
+ "name": "psi0_dance_audio",
41
+ "root_dir": "$PSI_DANCE_MOTION_ROOT",
42
+ "video_dir": "$PSI_DANCE_AUDIO_VIDEO_ROOT",
43
+ "dataset_name": "seed-audio-g1"
44
+ }
45
+ ]
46
+ JSON
47
+ )
48
+ fi
49
+
50
+ if [ -z "$PSI_AUDIO_COMMAND_SOURCES_JSON" ]; then
51
+ PSI_AUDIO_COMMAND_SOURCES_JSON=$(compact_json <<JSON
52
+ [
53
+ {
54
+ "name": "qwen3_tts_seed_audio_command",
55
+ "root_dir": "$PSI_SEED_ROOT",
56
+ "dataset_root": "$PSI_QWEN3_TTS_ROOT",
57
+ "metadata_jsonl": "metadata.jsonl",
58
+ "motion_dir_key": "filename",
59
+ "audio_path_key": "audio_path",
60
+ "instruction_key": "content_short_description",
61
+ "motion_metadata_csv": "$PSI_SEED_METADATA",
62
+ "motion_metadata_key": "filename",
63
+ "motion_metadata_path_key": "move_g1_path",
64
+ "audio_file_extensions": ".wav",
65
+ "audio_task": "command",
66
+ "dataset_name": "seed-audio-command-g1"
67
+ }
68
+ ]
69
+ JSON
70
+ )
71
+ fi
72
+
73
+ if [ -z "$PSI_VIDEO_SOURCES_JSON" ]; then
74
+ PSI_VIDEO_SOURCES_JSON=$(compact_json <<JSON
75
+ [
76
+ {
77
+ "name": "psi0_dance_gvhmr",
78
+ "root_dir": "$PSI_DANCE_MOTION_ROOT",
79
+ "video_dir": "$PSI_DANCE_VIDEO_ROOT",
80
+ "video_embed_dir": "$PSI_DANCE_VIDEO_EMBED_ROOT",
81
+ "dataset_name": "seed-video-g1"
82
+ },
83
+ {
84
+ "name": "motion_example_0506_demo",
85
+ "dataset_root": "$PSI_VIDEO_DEMO_ROOT",
86
+ "metadata_jsonl": "metadata.jsonl",
87
+ "motion_dir_key": "converted_path",
88
+ "video_path_key": "video_path",
89
+ "instruction_key": "video_caption",
90
+ "video_embed_dir": "$PSI_MOTION_EXAMPLE_VIDEO_EMBED_ROOT",
91
+ "dataset_name": "motion-example-video-g1"
92
+ }
93
+ ]
94
+ JSON
95
+ )
96
+ fi
97
+
98
+ if [ -z "$PSI_VIDEO_CLIP_SOURCES_JSON" ]; then
99
+ PSI_VIDEO_CLIP_SOURCES_JSON=$(compact_json <<JSON
100
+ [
101
+ {
102
+ "name": "dance_1723_video_10s_clip50",
103
+ "dataset_root": "$PSI_DANCE_VIDEO_CLIP_ROOT",
104
+ "metadata_jsonl": "metadata.jsonl",
105
+ "motion_dir_key": "converted_path",
106
+ "video_path_key": "video_path",
107
+ "instruction_key": "video_caption",
108
+ "video_embed_dir": "$PSI_DANCE_VIDEO_CLIP_ROOT/embeds",
109
+ "dataset_name": "dance-video-clip-g1"
110
+ },
111
+ {
112
+ "name": "motion_example_0506_demo_clip50",
113
+ "dataset_root": "$PSI_MOTION_EXAMPLE_VIDEO_CLIP_ROOT",
114
+ "metadata_jsonl": "metadata.jsonl",
115
+ "motion_dir_key": "converted_path",
116
+ "video_path_key": "video_path",
117
+ "instruction_key": "video_caption",
118
+ "video_embed_dir": "$PSI_MOTION_EXAMPLE_VIDEO_CLIP_ROOT/embeds",
119
+ "dataset_name": "motion-example-video-clip-g1"
120
+ }
121
+ ]
122
+ JSON
123
+ )
124
+ fi
125
+
126
+ PSI_RESUME_TEXT_CHECKPOINT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/Psi0-yth-v18/runs/posttrain_v1_trimodal/posttrain.sg.flow1000.cwml.lr1.0e-04.b256.gpus8.2605310619/checkpoints/ckpt_280000
127
+
128
+ PSI_TEXT_TASK_WEIGHT=0.5
129
+ PSI_AUDIO_TEXT_TASK_WEIGHT=0
130
+ PSI_AUDIO_ONLY_TASK_WEIGHT=0
131
+ PSI_AUDIO_COMMAND_TEXT_TASK_WEIGHT=0
132
+ PSI_AUDIO_COMMAND_ONLY_TASK_WEIGHT=0
133
+ PSI_VIDEO_TEXT_TASK_WEIGHT=0
134
+ PSI_VIDEO_ONLY_TASK_WEIGHT=0
135
+ PSI_VIDEO_CLIP_TEXT_TASK_WEIGHT=0.25
136
+ PSI_VIDEO_CLIP_ONLY_TASK_WEIGHT=0.25
137
+
138
+ PSI_PRETRAIN_BACKBONE_PATH=/cpfs01/VideoGen/public_ckpt/Qwen/Qwen2.5-Omni-3B
139
+ NUM_PAST_FRAMES=0
140
+ STATE_HORIZON=$((NUM_PAST_FRAMES + 1))
141
+ ACTION_CHUNK_SIZE=30
142
+ ACTION_EXEC_HORIZON=20
143
+ PROGRESS_DIM=5
144
+ VIDEO_SAMPLE_FPS=1.0
145
+ VIDEO_CLIP_SAMPLE_FPS=8.0
146
+ VIDEO_SIZE_PIXELS=50176
147
+ VIDEO_DECODER=ffmpeg
148
+ VIDEO_CACHE_SIZE=64
149
+ DATALOADER_NUM_WORKERS=4
150
+ VAL_DATALOADER_NUM_WORKERS=2
151
+
152
+ if [ ! -f "$PSI_RESUME_TEXT_CHECKPOINT/model.safetensors" ] || [ ! -f "$PSI_RESUME_TEXT_CHECKPOINT/optimizer.bin" ] || [ ! -f "$PSI_RESUME_TEXT_CHECKPOINT/scheduler.bin" ]; then
153
+ echo "ERROR: resume checkpoint is not a full training-state checkpoint: $PSI_RESUME_TEXT_CHECKPOINT" >&2
154
+ exit 1
155
+ fi
156
+
157
+ NPROC_PER_NODE=$(echo $CUDA_VISIBLE_DEVICES | tr ',' '\n' | wc -l)
158
+
159
+ echo "Training with $NPROC_PER_NODE GPU(s): $CUDA_VISIBLE_DEVICES"
160
+ echo "Text root: $PSI_SEED_ROOT"
161
+ echo "Dance motion root: $PSI_DANCE_MOTION_ROOT"
162
+ echo "Dance audio/video root: $PSI_DANCE_AUDIO_VIDEO_ROOT"
163
+ echo "Dance video root: $PSI_DANCE_VIDEO_ROOT"
164
+ echo "Dance video embed root: $PSI_DANCE_VIDEO_EMBED_ROOT"
165
+ echo "Qwen3-TTS acdudio command root: $PSI_QWEN3_TTS_ROOT"
166
+ echo "Audio sources: $PSI_AUDIO_SOURCES_JSON"
167
+ echo "Audio command sources: $PSI_AUDIO_COMMAND_SOURCES_JSON"
168
+ echo "Video sources: $PSI_VIDEO_SOURCES_JSON"
169
+ echo "Video clip sources: $PSI_VIDEO_CLIP_SOURCES_JSON"
170
+ echo "Task weights: text=$PSI_TEXT_TASK_WEIGHT audio_text=$PSI_AUDIO_TEXT_TASK_WEIGHT audio_only=$PSI_AUDIO_ONLY_TASK_WEIGHT audio_command_text=$PSI_AUDIO_COMMAND_TEXT_TASK_WEIGHT audio_command_only=$PSI_AUDIO_COMMAND_ONLY_TASK_WEIGHT video_text=$PSI_VIDEO_TEXT_TASK_WEIGHT video_only=$PSI_VIDEO_ONLY_TASK_WEIGHT video_clip_text=$PSI_VIDEO_CLIP_TEXT_TASK_WEIGHT video_clip_only=$PSI_VIDEO_CLIP_ONLY_TASK_WEIGHT"
171
+ echo "Resume full training state from: $PSI_RESUME_TEXT_CHECKPOINT"
172
+ echo "TextOp 39D action/state enabled"
173
+ echo "Future progress embedding: enabled, dim $PROGRESS_DIM"
174
+ echo "Backbone path: $PSI_PRETRAIN_BACKBONE_PATH"
175
+ echo "Output dir: $PSI_OUTPUT_DIR"
176
+
177
+ args="posttrain_v1_seed_text_audio_video_mix_psi0_config \
178
+ --seed=292285 \
179
+ --exp=posttrain \
180
+ --timestamp=$(date +"%y%m%d%H%M") \
181
+ --train.name=posttrain_v1_trimodal \
182
+ --train.data_parallel=ddp \
183
+ --train.mixed_precision=bf16 \
184
+ --train.train_batch_size=32 \
185
+ --train.val_batch_size=64 \
186
+ --train.num-workers=$DATALOADER_NUM_WORKERS \
187
+ --train.val-num-workers=$VAL_DATALOADER_NUM_WORKERS \
188
+ --train.resume_from_checkpoint=$PSI_RESUME_TEXT_CHECKPOINT \
189
+ --train.resume-load-full-state \
190
+ --train.max_checkpoints_to_keep=1 \
191
+ --train.gradient_accumulation_steps=1 \
192
+ --train.learning_rate=1e-4 \
193
+ --train.max_training_steps=500000 \
194
+ --train.warmup_ratio=None \
195
+ --train.warmup_steps=1000 \
196
+ --train.checkpointing_steps=1000 \
197
+ --train.validation_steps=1000 \
198
+ --train.val_num_batches=20 \
199
+ --train.max_grad_norm=1.0 \
200
+ --train.lr_scheduler_type=cosine_with_min_lr \
201
+ --train.lr_scheduler_kwargs.betas 0.9 0.999 \
202
+ --train.lr_scheduler_kwargs.weight_decay=0 \
203
+ --train.lr_scheduler_kwargs.eps=1e-8 \
204
+ --log.report_to=$PSI_LOG_REPORT_TO \
205
+ --train.output_dir=$PSI_OUTPUT_DIR \
206
+ --data.text.root-dir=$PSI_SEED_ROOT \
207
+ --data.text.metadata-csv=$PSI_SEED_METADATA \
208
+ --data.text-ratio=$PSI_TEXT_TASK_WEIGHT \
209
+ --data.audio.sources-json=$PSI_AUDIO_SOURCES_JSON \
210
+ --data.audio-ratio=$PSI_AUDIO_TEXT_TASK_WEIGHT \
211
+ --data.audio-only-ratio=$PSI_AUDIO_ONLY_TASK_WEIGHT \
212
+ --data.audio.sample-stride=1 \
213
+ --data.audio.motion-fps=50 \
214
+ --data.audio.audio-sampling-rate=16000 \
215
+ --data.audio.audio-window-seconds=1.0 \
216
+ --data.audio.audio-left-context-seconds=0.0 \
217
+ --data.audio.full-audio \
218
+ --data.audio-command.sources-json=$PSI_AUDIO_COMMAND_SOURCES_JSON \
219
+ --data.audio-command-ratio=$PSI_AUDIO_COMMAND_TEXT_TASK_WEIGHT \
220
+ --data.audio-command-only-ratio=$PSI_AUDIO_COMMAND_ONLY_TASK_WEIGHT \
221
+ --data.audio-command.sample-stride=1 \
222
+ --data.audio-command.motion-fps=50 \
223
+ --data.audio-command.audio-sampling-rate=16000 \
224
+ --data.audio-command.audio-window-seconds=1.0 \
225
+ --data.audio-command.audio-left-context-seconds=0.0 \
226
+ --data.audio-command.full-audio \
227
+ --data.video-ratio=$PSI_VIDEO_TEXT_TASK_WEIGHT \
228
+ --data.video-only-ratio=$PSI_VIDEO_ONLY_TASK_WEIGHT \
229
+ --data.video.sample-stride=1 \
230
+ --data.video.motion-fps=50 \
231
+ --data.video.video-sample-fps=$VIDEO_SAMPLE_FPS \
232
+ --data.video.video-size-pixels=$VIDEO_SIZE_PIXELS \
233
+ --data.video.video-decoder=$VIDEO_DECODER \
234
+ --data.video.video-cache-size=$VIDEO_CACHE_SIZE \
235
+ --data.video.sources-json=$PSI_VIDEO_SOURCES_JSON \
236
+ --data.video-clip-text-ratio=$PSI_VIDEO_CLIP_TEXT_TASK_WEIGHT \
237
+ --data.video-clip-only-ratio=$PSI_VIDEO_CLIP_ONLY_TASK_WEIGHT \
238
+ --data.video-clip.sources-json=$PSI_VIDEO_CLIP_SOURCES_JSON \
239
+ --data.video-clip.sample-stride=1 \
240
+ --data.video-clip.motion-fps=50 \
241
+ --data.video-clip.video-sample-fps=$VIDEO_CLIP_SAMPLE_FPS \
242
+ --data.video-clip.video-size-pixels=$VIDEO_SIZE_PIXELS \
243
+ --data.video-clip.video-decoder=$VIDEO_DECODER \
244
+ --data.video-clip.video-cache-size=$VIDEO_CACHE_SIZE \
245
+ --data.video-clip.action-exec-horizon=$ACTION_EXEC_HORIZON \
246
+ --data.sampler=batch_mixture \
247
+ --data.val-task=text,video_clip,video_clip_only \
248
+ --data.transform.repack.num-past-frames=$NUM_PAST_FRAMES \
249
+ --data.transform.repack.textop-actions \
250
+ --data.transform.repack.action-chunk-size=$ACTION_CHUNK_SIZE \
251
+ --data.transform.repack.pad-action-dim=39 \
252
+ --data.transform.repack.pad-state-dim=39 \
253
+ --data.transform.field.action_norm_type=bounds_q99 \
254
+ --data.transform.field.stat-path=assets/stats/seed_all_g1_motion39_textop_stats.json \
255
+ --data.transform.field.normalize-state \
256
+ --data.transform.field.pad-action-dim=39 \
257
+ --data.transform.field.pad-state-dim=39 \
258
+ --data.transform.model.resize.size 240 320 \
259
+ --data.transform.model.center_crop.size 240 320 \
260
+ --data.transform.model.no-img-aug \
261
+ --data.transform.model.text-only \
262
+ --data.transform.model.audio-only \
263
+ --data.transform.model.video-only \
264
+ --data.transform.model.video-size-pixels=$VIDEO_SIZE_PIXELS \
265
+ --model.model_name_or_path=$PSI_PRETRAIN_BACKBONE_PATH \
266
+ --model.noise-scheduler=flow \
267
+ --model.n_conditions=0 \
268
+ --model.action-chunk-size=$ACTION_CHUNK_SIZE \
269
+ --model.action-dim=39 \
270
+ --model.action-exec-horizon=$ACTION_EXEC_HORIZON \
271
+ --model.observation-horizon=$STATE_HORIZON \
272
+ --model.odim=39 \
273
+ --model.view_feature_dim=2048 \
274
+ --model.use-future-progress-embedding \
275
+ --model.future-progress-dim=$PROGRESS_DIM \
276
+ --model.tune-vlm \
277
+ --model.lang-backbone-lr=1e-5 \
278
+ --model.no-use_film \
279
+ --model.no-combined_temb \
280
+ --model.rtc \
281
+ --model.max-delay=11 \
282
+ --train.max-grad-value=1 \
283
+ --train.loss-group-weight.joint-abs=1.0 \
284
+ --train.loss-group-weight.root-delta-xy=3.0 \
285
+ --train.loss-group-weight.root-rot5-delta=10.0 \
286
+ --train.loss-group-weight.root-height=15.0 \
287
+ --train.loss-group-weight.foot-contact=0.1
288
+ "
289
+ torchrun --nproc_per_node=$NPROC_PER_NODE --master_port=29500 scripts/train.py \
290
+ $args "$@"
prebuild-multimodal-records-cache.sh ADDED
@@ -0,0 +1,280 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ cd /cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/Psi0-yth-v21
2
+ export PYTHONPATH=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/Psi0-yth-v21/src
3
+
4
+ source /cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/Psi0-yth-v15/.venv-psi/bin/activate
5
+ [ -f .env ] && source .env
6
+
7
+ export OMP_NUM_THREADS=${OMP_NUM_THREADS:-8}
8
+ export TF_CPP_MIN_LOG_LEVEL=${TF_CPP_MIN_LOG_LEVEL:-3}
9
+
10
+ compact_json() {
11
+ python -c 'import json, sys; print(json.dumps(json.load(sys.stdin), separators=(",", ":")))'
12
+ }
13
+
14
+ PSI_SEED_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/Psi0/dataset/seed_all
15
+ PSI_SEED_METADATA=metadata/seed_metadata_v004.csv
16
+ PSI_TEXT_RECORDS_CACHE_DIR=${PSI_TEXT_RECORDS_CACHE_DIR:-$PSI_SEED_ROOT/.index_cache_multimodal_yth}
17
+
18
+ PSI_DANCE_MOTION_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/GR00T-WholeBodyControl/gear_sonic_deploy/reference/converted/psi0_dance/gvhmr
19
+ PSI_DANCE_AUDIO_VIDEO_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/dance_1723_video_psi0/dance_1723_video_10s
20
+ PSI_DANCE_VIDEO_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/dance_1723_video_psi0/dance_1723_video_10s
21
+ PSI_DANCE_VIDEO_EMBED_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/dance_1723_video_psi0/dance_1723_video_10s_embeds
22
+ PSI_VIDEO_DEMO_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/videogen/motion_example_0506_demo
23
+ PSI_2264_WITH_AUDIO_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/videogen/2264_with_audio
24
+ PSI_QWEN3_TTS_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/qwen3-tts
25
+ PSI_VIDEO_CLIP_DATASET_ROOT=${PSI_VIDEO_CLIP_DATASET_ROOT:-/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/videogen/psi0_video_clip_50f_dense_stride1_direct_full}
26
+ PSI_DANCE_VIDEO_CLIP_ROOT=$PSI_VIDEO_CLIP_DATASET_ROOT/dance_1723_video_10s
27
+ PSI_MOTION_EXAMPLE_VIDEO_CLIP_ROOT=$PSI_VIDEO_CLIP_DATASET_ROOT/motion_example_0506_demo
28
+ PSI_2264_WITH_AUDIO_VIDEO_CLIP_ROOT=$PSI_VIDEO_CLIP_DATASET_ROOT/2264_with_audio
29
+ PSI_DANCE_VIDEO_CLIP_EMBED_ROOT=${PSI_DANCE_VIDEO_CLIP_EMBED_ROOT:-$PSI_DANCE_VIDEO_CLIP_ROOT/embeds}
30
+ PSI_MOTION_EXAMPLE_VIDEO_CLIP_EMBED_ROOT=${PSI_MOTION_EXAMPLE_VIDEO_CLIP_EMBED_ROOT:-$PSI_MOTION_EXAMPLE_VIDEO_CLIP_ROOT/embeds}
31
+ PSI_2264_WITH_AUDIO_VIDEO_CLIP_EMBED_ROOT=${PSI_2264_WITH_AUDIO_VIDEO_CLIP_EMBED_ROOT:-$PSI_2264_WITH_AUDIO_VIDEO_CLIP_ROOT/embeds}
32
+ PSI_MOTION_EXAMPLE_VIDEO_EMBED_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/videogen/motion_example_0506_demo_video_embeds
33
+ PSI_AUDIO_RECORDS_CACHE_DIR=${PSI_AUDIO_RECORDS_CACHE_DIR:-$PSI_2264_WITH_AUDIO_ROOT/.index_cache_multimodal_yth}
34
+ PSI_AUDIO_COMMAND_RECORDS_CACHE_DIR=${PSI_AUDIO_COMMAND_RECORDS_CACHE_DIR:-$PSI_QWEN3_TTS_ROOT/.index_cache_multimodal_yth}
35
+ PSI_DANCE_AUDIO_NPY_ROOT=${PSI_DANCE_AUDIO_NPY_ROOT:-$PSI_DANCE_AUDIO_VIDEO_ROOT/audio_npy_yth}
36
+ PSI_DANCE_AUDIO_NPY_MANIFEST=${PSI_DANCE_AUDIO_NPY_MANIFEST:-$PSI_DANCE_AUDIO_VIDEO_ROOT/metadata_audio_npy.jsonl}
37
+ PSI_2264_AUDIO_NPY_MANIFEST=${PSI_2264_AUDIO_NPY_MANIFEST:-$PSI_2264_WITH_AUDIO_ROOT/metadata_audio_npy.jsonl}
38
+ PSI_QWEN3_TTS_AUDIO_NPY_ROOT=${PSI_QWEN3_TTS_AUDIO_NPY_ROOT:-$PSI_QWEN3_TTS_ROOT/audio_npy_yth}
39
+ PSI_QWEN3_TTS_AUDIO_NPY_MANIFEST=${PSI_QWEN3_TTS_AUDIO_NPY_MANIFEST:-$PSI_QWEN3_TTS_ROOT/metadata_audio_npy.jsonl}
40
+
41
+ if [ -f "$PSI_2264_AUDIO_NPY_MANIFEST" ]; then
42
+ PSI_2264_AUDIO_METADATA_JSONL=$PSI_2264_AUDIO_NPY_MANIFEST
43
+ PSI_2264_AUDIO_PATH_KEY=audio_path
44
+ else
45
+ PSI_2264_AUDIO_METADATA_JSONL=metadata_convert.jsonl
46
+ PSI_2264_AUDIO_PATH_KEY=video_path
47
+ fi
48
+
49
+ if [ -f "$PSI_DANCE_AUDIO_NPY_MANIFEST" ]; then
50
+ PSI_DANCE_AUDIO_SOURCE_JSON=$(cat <<JSON
51
+ {
52
+ "name": "psi0_dance_audio",
53
+ "dataset_root": "$PSI_DANCE_AUDIO_VIDEO_ROOT",
54
+ "metadata_jsonl": "$PSI_DANCE_AUDIO_NPY_MANIFEST",
55
+ "motion_dir_key": "converted_path",
56
+ "audio_path_key": "audio_path",
57
+ "records_cache_dir": "$PSI_AUDIO_RECORDS_CACHE_DIR",
58
+ "dataset_name": "seed-audio-g1"
59
+ }
60
+ JSON
61
+ )
62
+ else
63
+ PSI_DANCE_AUDIO_SOURCE_JSON=$(cat <<JSON
64
+ {
65
+ "name": "psi0_dance_audio",
66
+ "root_dir": "$PSI_DANCE_MOTION_ROOT",
67
+ "video_dir": "$PSI_DANCE_AUDIO_VIDEO_ROOT",
68
+ "records_cache_dir": "$PSI_AUDIO_RECORDS_CACHE_DIR",
69
+ "dataset_name": "seed-audio-g1"
70
+ }
71
+ JSON
72
+ )
73
+ fi
74
+
75
+ if [ -z "$PSI_AUDIO_SOURCES_JSON" ]; then
76
+ PSI_AUDIO_SOURCES_JSON=$(compact_json <<JSON
77
+ [
78
+ $PSI_DANCE_AUDIO_SOURCE_JSON,
79
+ {
80
+ "name": "2264_with_audio",
81
+ "dataset_root": "$PSI_2264_WITH_AUDIO_ROOT",
82
+ "metadata_jsonl": "$PSI_2264_AUDIO_METADATA_JSONL",
83
+ "motion_dir_key": "converted_path",
84
+ "audio_path_key": "$PSI_2264_AUDIO_PATH_KEY",
85
+ "records_cache_dir": "$PSI_AUDIO_RECORDS_CACHE_DIR",
86
+ "dataset_name": "2264-with-audio-g1"
87
+ }
88
+ ]
89
+ JSON
90
+ )
91
+ fi
92
+
93
+ if [ -f "$PSI_QWEN3_TTS_AUDIO_NPY_MANIFEST" ]; then
94
+ PSI_AUDIO_COMMAND_SOURCE_JSON=$(cat <<JSON
95
+ {
96
+ "name": "qwen3_tts_seed_audio_command",
97
+ "root_dir": "$PSI_SEED_ROOT",
98
+ "dataset_root": "$PSI_QWEN3_TTS_ROOT",
99
+ "metadata_jsonl": "$PSI_QWEN3_TTS_AUDIO_NPY_MANIFEST",
100
+ "motion_dir_key": "filename",
101
+ "audio_path_key": "audio_path",
102
+ "instruction_key": "content_short_description",
103
+ "motion_metadata_csv": "$PSI_SEED_METADATA",
104
+ "motion_metadata_key": "filename",
105
+ "motion_metadata_path_key": "move_g1_path",
106
+ "audio_file_extensions": ".npy",
107
+ "records_cache_dir": "$PSI_AUDIO_COMMAND_RECORDS_CACHE_DIR",
108
+ "audio_task": "command",
109
+ "dataset_name": "seed-audio-command-g1"
110
+ }
111
+ JSON
112
+ )
113
+ else
114
+ PSI_AUDIO_COMMAND_SOURCE_JSON=$(cat <<JSON
115
+ {
116
+ "name": "qwen3_tts_seed_audio_command",
117
+ "root_dir": "$PSI_SEED_ROOT",
118
+ "dataset_root": "$PSI_QWEN3_TTS_ROOT",
119
+ "metadata_jsonl": "metadata.jsonl",
120
+ "motion_dir_key": "filename",
121
+ "audio_path_key": "audio_path",
122
+ "instruction_key": "content_short_description",
123
+ "motion_metadata_csv": "$PSI_SEED_METADATA",
124
+ "motion_metadata_key": "filename",
125
+ "motion_metadata_path_key": "move_g1_path",
126
+ "audio_file_extensions": ".wav",
127
+ "records_cache_dir": "$PSI_AUDIO_COMMAND_RECORDS_CACHE_DIR",
128
+ "audio_task": "command",
129
+ "dataset_name": "seed-audio-command-g1"
130
+ }
131
+ JSON
132
+ )
133
+ fi
134
+
135
+ if [ -z "$PSI_AUDIO_COMMAND_SOURCES_JSON" ]; then
136
+ PSI_AUDIO_COMMAND_SOURCES_JSON=$(compact_json <<JSON
137
+ [
138
+ $PSI_AUDIO_COMMAND_SOURCE_JSON
139
+ ]
140
+ JSON
141
+ )
142
+ fi
143
+
144
+ if [ -z "$PSI_VIDEO_SOURCES_JSON" ]; then
145
+ PSI_VIDEO_SOURCES_JSON=$(compact_json <<JSON
146
+ [
147
+ {
148
+ "name": "psi0_dance_gvhmr",
149
+ "root_dir": "$PSI_DANCE_MOTION_ROOT",
150
+ "video_dir": "$PSI_DANCE_VIDEO_ROOT",
151
+ "video_embed_dir": "$PSI_DANCE_VIDEO_EMBED_ROOT",
152
+ "dataset_name": "seed-video-g1"
153
+ },
154
+ {
155
+ "name": "motion_example_0506_demo",
156
+ "dataset_root": "$PSI_VIDEO_DEMO_ROOT",
157
+ "metadata_jsonl": "metadata.jsonl",
158
+ "motion_dir_key": "converted_path",
159
+ "video_path_key": "video_path",
160
+ "instruction_key": "video_caption",
161
+ "video_embed_dir": "$PSI_MOTION_EXAMPLE_VIDEO_EMBED_ROOT",
162
+ "dataset_name": "motion-example-video-g1"
163
+ }
164
+ ]
165
+ JSON
166
+ )
167
+ fi
168
+
169
+ if [ -z "$PSI_VIDEO_CLIP_SOURCES_JSON" ]; then
170
+ PSI_VIDEO_CLIP_SOURCES_JSON=$(compact_json <<JSON
171
+ [
172
+ {
173
+ "name": "dance_1723_video_10s_clip50",
174
+ "dataset_root": "$PSI_DANCE_VIDEO_CLIP_ROOT",
175
+ "metadata_jsonl": "metadata.jsonl",
176
+ "motion_dir_key": "converted_path",
177
+ "video_path_key": "video_path",
178
+ "instruction_key": "video_caption",
179
+ "video_embed_dir": "$PSI_DANCE_VIDEO_CLIP_EMBED_ROOT",
180
+ "dataset_name": "dance-video-clip-g1"
181
+ },
182
+ {
183
+ "name": "motion_example_0506_demo_clip50",
184
+ "dataset_root": "$PSI_MOTION_EXAMPLE_VIDEO_CLIP_ROOT",
185
+ "metadata_jsonl": "metadata.jsonl",
186
+ "motion_dir_key": "converted_path",
187
+ "video_path_key": "video_path",
188
+ "instruction_key": "video_caption",
189
+ "video_embed_dir": "$PSI_MOTION_EXAMPLE_VIDEO_CLIP_EMBED_ROOT",
190
+ "dataset_name": "motion-example-video-clip-g1"
191
+ },
192
+ {
193
+ "name": "2264_with_audio_clip50",
194
+ "dataset_root": "$PSI_2264_WITH_AUDIO_VIDEO_CLIP_ROOT",
195
+ "metadata_jsonl": "metadata.jsonl",
196
+ "motion_dir_key": "converted_path",
197
+ "video_path_key": "video_path",
198
+ "instruction_key": "video_caption",
199
+ "video_embed_dir": "$PSI_2264_WITH_AUDIO_VIDEO_CLIP_EMBED_ROOT",
200
+ "dataset_name": "2264-with-audio-video-clip-g1"
201
+ }
202
+ ]
203
+ JSON
204
+ )
205
+ fi
206
+
207
+ ACTION_CHUNK_SIZE=${ACTION_CHUNK_SIZE:-30}
208
+ ACTION_EXEC_HORIZON=${ACTION_EXEC_HORIZON:-20}
209
+ NUM_PAST_FRAMES=${NUM_PAST_FRAMES:-0}
210
+ VIDEO_SAMPLE_FPS=${VIDEO_SAMPLE_FPS:-1.0}
211
+ VIDEO_CLIP_SAMPLE_FPS=${VIDEO_CLIP_SAMPLE_FPS:-8.0}
212
+ VIDEO_SIZE_PIXELS=${VIDEO_SIZE_PIXELS:-50176}
213
+ VIDEO_DECODER=${VIDEO_DECODER:-ffmpeg}
214
+ VIDEO_RECORD_BUILD_WORKERS=${VIDEO_RECORD_BUILD_WORKERS:-32}
215
+ VIDEO_RECORD_BUILD_CHUNK_SIZE=${VIDEO_RECORD_BUILD_CHUNK_SIZE:-4096}
216
+ VIDEO_CLIP_TRUST_METADATA=${VIDEO_CLIP_TRUST_METADATA:-1}
217
+ VIDEO_CLIP_RECORDS_CACHE_DIR=${VIDEO_CLIP_RECORDS_CACHE_DIR:-$PSI_VIDEO_CLIP_DATASET_ROOT/.index_cache_multimodal_yth}
218
+ PREBUILD_TASKS=${PREBUILD_TASKS:-text,audio,audio_command,video_clip}
219
+ PREBUILD_SPLITS=${PREBUILD_SPLITS:-train,val}
220
+
221
+ if [ "$VIDEO_CLIP_TRUST_METADATA" = "1" ] || [ "$VIDEO_CLIP_TRUST_METADATA" = "true" ] || [ "$VIDEO_CLIP_TRUST_METADATA" = "True" ]; then
222
+ PREBUILD_VIDEO_CLIP_TRUST_METADATA_FLAG="--video-clip-trust-metadata"
223
+ else
224
+ PREBUILD_VIDEO_CLIP_TRUST_METADATA_FLAG="--no-video-clip-trust-metadata"
225
+ fi
226
+
227
+ echo "Prebuild tasks: $PREBUILD_TASKS"
228
+ echo "Prebuild splits: $PREBUILD_SPLITS"
229
+ echo "Text cache dir: $PSI_TEXT_RECORDS_CACHE_DIR"
230
+ echo "Audio cache dir: $PSI_AUDIO_RECORDS_CACHE_DIR"
231
+ echo "Audio command cache dir: $PSI_AUDIO_COMMAND_RECORDS_CACHE_DIR"
232
+ echo "Dance audio manifest: $PSI_DANCE_AUDIO_NPY_MANIFEST"
233
+ echo "2264 audio manifest: $PSI_2264_AUDIO_METADATA_JSONL"
234
+ echo "Qwen3-TTS audio manifest: $PSI_QWEN3_TTS_AUDIO_NPY_MANIFEST"
235
+ echo "Video clip cache dir: $VIDEO_CLIP_RECORDS_CACHE_DIR"
236
+ echo "Video record build workers: $VIDEO_RECORD_BUILD_WORKERS"
237
+ echo "Video record build chunk size: $VIDEO_RECORD_BUILD_CHUNK_SIZE"
238
+
239
+ python scripts/data/prebuild_multimodal_records_cache.py \
240
+ --tasks="$PREBUILD_TASKS" \
241
+ --splits="$PREBUILD_SPLITS" \
242
+ --action-chunk-size=$ACTION_CHUNK_SIZE \
243
+ --num-past-frames=$NUM_PAST_FRAMES \
244
+ --text-root-dir="$PSI_SEED_ROOT" \
245
+ --text-metadata-csv="$PSI_SEED_METADATA" \
246
+ --text-records-cache-dir="$PSI_TEXT_RECORDS_CACHE_DIR" \
247
+ --audio-sources-json="$PSI_AUDIO_SOURCES_JSON" \
248
+ --audio-records-cache-dir="$PSI_AUDIO_RECORDS_CACHE_DIR" \
249
+ --audio-sample-stride=1 \
250
+ --audio-motion-fps=50 \
251
+ --audio-audio-sampling-rate=16000 \
252
+ --audio-audio-window-seconds=1.0 \
253
+ --audio-audio-left-context-seconds=0.0 \
254
+ --audio-command-sources-json="$PSI_AUDIO_COMMAND_SOURCES_JSON" \
255
+ --audio-command-records-cache-dir="$PSI_AUDIO_COMMAND_RECORDS_CACHE_DIR" \
256
+ --audio-command-sample-stride=1 \
257
+ --audio-command-motion-fps=50 \
258
+ --audio-command-audio-sampling-rate=16000 \
259
+ --audio-command-audio-window-seconds=1.0 \
260
+ --audio-command-audio-left-context-seconds=0.0 \
261
+ --video-sources-json="$PSI_VIDEO_SOURCES_JSON" \
262
+ --video-sample-stride=1 \
263
+ --video-motion-fps=50 \
264
+ --video-video-sample-fps=$VIDEO_SAMPLE_FPS \
265
+ --video-video-size-pixels=$VIDEO_SIZE_PIXELS \
266
+ --video-video-decoder=$VIDEO_DECODER \
267
+ --video-record-build-workers=$VIDEO_RECORD_BUILD_WORKERS \
268
+ --video-record-build-chunk-size=$VIDEO_RECORD_BUILD_CHUNK_SIZE \
269
+ --video-clip-sources-json="$PSI_VIDEO_CLIP_SOURCES_JSON" \
270
+ --video-clip-records-cache-dir="$VIDEO_CLIP_RECORDS_CACHE_DIR" \
271
+ --video-clip-sample-stride=1 \
272
+ --video-clip-motion-fps=50 \
273
+ --video-clip-video-sample-fps=$VIDEO_CLIP_SAMPLE_FPS \
274
+ --video-clip-video-size-pixels=$VIDEO_SIZE_PIXELS \
275
+ --video-clip-video-decoder=$VIDEO_DECODER \
276
+ --video-clip-record-build-workers=$VIDEO_RECORD_BUILD_WORKERS \
277
+ --video-clip-record-build-chunk-size=$VIDEO_RECORD_BUILD_CHUNK_SIZE \
278
+ $PREBUILD_VIDEO_CLIP_TRUST_METADATA_FLAG \
279
+ --video-clip-action-exec-horizon=$ACTION_EXEC_HORIZON \
280
+ "$@"
pyproject.toml ADDED
@@ -0,0 +1,85 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ [project]
2
+ name = "psi"
3
+ version = "0.0.0"
4
+ description = "Psi: towards humanoid foundation model"
5
+ readme = "README.md"
6
+ requires-python = "==3.10.*"
7
+ dependencies = [
8
+ "albumentations==1.4.18",
9
+ "datasets==3.6.0",
10
+ "dm-tree==0.1.8",
11
+ "einops==0.8.1",
12
+ "einx>=0.3.0",
13
+ "h5py==3.14.0",
14
+ "imageio==2.34.2",
15
+ "jsonlines>=4.0.0",
16
+ "numpy<2.0.0",
17
+ "numpydantic==1.6.7",
18
+ "omegaconf>=2.3.0",
19
+ "opencv-python",
20
+ "plotly==6.2.0",
21
+ "pydantic==2.10.6",
22
+ "pydantic-yaml>=1.6.0",
23
+ "tqdm==4.67.1",
24
+ "typer>=0.19.1",
25
+ "tyro==0.9.32",
26
+ "vuer[all]>=0.0.68",
27
+ "wandb>=0.20.0",
28
+ "simplejpeg>=1.9.0",
29
+ "dotenv",
30
+ "gdown>=5.2.0",
31
+ ]
32
+
33
+ [build-system]
34
+ requires = ["setuptools>=61.0"]
35
+ build-backend = "setuptools.build_meta"
36
+
37
+ [tool.setuptools.package-data]
38
+ psi = ["assets/*"]
39
+
40
+ [tool.uv.sources]
41
+ psi = { path = ".", editable = true }
42
+ simple = { path = "third_party/SIMPLE", editable = true }
43
+
44
+ [tool.uv.workspace]
45
+ members = ["packages/*"]
46
+
47
+ [dependency-groups]
48
+ serve = [
49
+ "fastapi",
50
+ "uvicorn",
51
+ ]
52
+
53
+ viz = [
54
+ "transforms3d>=0.4.2",
55
+ "vuer[all]>=0.0.68",
56
+ "viser>=1.0.15",
57
+ #"pin>=3.8.0",
58
+ #"numpy<2.0.0",
59
+ ]
60
+
61
+ psi = [
62
+ "setuptools>=61",
63
+ "torch==2.7.0",
64
+ "torchvision==0.22.0",
65
+ "torchcodec==0.4.0",
66
+ "transformers==4.57.0",
67
+ "deepspeed==0.17.1",
68
+ "flash_attn==2.7.4.post1",
69
+ "triton==3.3.0",
70
+ "accelerate==1.7.0",
71
+ "peft==0.17.1",
72
+ "qwen-vl-utils==0.0.14",
73
+ "sentencepiece",
74
+ "lerobot @ git+https://github.com/songlin/lerobot.git@09929d8057b044b53aecaf5c6d7eb71f99e8beb9"
75
+ ]
76
+
77
+ #simple = [
78
+ # "simple[full]",
79
+ #]
80
+
81
+ [tool.uv.extra-build-dependencies]
82
+ flash-attn = [{ requirement = "torch", match-runtime = true }]
83
+
84
+ [project.scripts]
85
+ serve_psi0 = "psi.deploy.psi0_serve_simple:main"
uv.lock ADDED
The diff for this file is too large to render. See raw diff