Add files using upload-large-folder tool
Browse files- .env.sample +16 -0
- .gitignore +71 -0
- .gitmodules +3 -0
- 26072112-reuse6-task3131.md +62 -0
- 2608121521-conclusion.md +237 -0
- 260904-kv_memory.md +468 -0
- 260904-lzk交接文档.md +447 -0
- 260925-third-party人声重点版更新报告.md +66 -0
- 260926-third-party中英交叉版更新报告.md +53 -0
- 260927-third-party数据与GT速览.md +44 -0
- 260927-third-party新版GT审核与清理.md +68 -0
- 260927-third-party背景音量修复.md +41 -0
- 260928-third-party多段人声更新报告.md +83 -0
- 260930-third-party训练实验计划.md +68 -0
- 261002-third-party训练实验操作手册.md +232 -0
- AGENTS.md +29 -0
- CHANGELOG.md +0 -0
- DIALOG.md +0 -0
- EXPERIMENT.md +394 -0
- FULL_DUPLEX_TASK_RESEARCH.md +120 -0
- LICENSE +17 -0
- OMNI_DATA_PREPROCESS.md +128 -0
- Psi0-训练阶段-state-action总览.md +216 -0
- README.md +532 -0
- flake.lock +27 -0
- infer.sh +41 -0
- infer_prompt.sh +214 -0
- infer_prompt_distill.sh +105 -0
- infer_prompt_short.sh +158 -0
- infer_render.sh +65 -0
- infer_render_auto.sh +454 -0
- plan_0.md +374 -0
- posttrain-full-duplex-audio-command-canonical-motion.sh +27 -0
- posttrain-full-duplex-audio-command-overfit.sh +43 -0
- posttrain-full-duplex-audio-command-smooth.sh +20 -0
- posttrain-full-duplex-audio-command.sh +1523 -0
- posttrain-v1-psi0-all-7task-cached-oss.sh +775 -0
- posttrain-v1-psi0-all-7task-cached.sh +459 -0
- posttrain-v1-psi0-all-7task-tune-vlm-direct-dense-actionhead-L.sh +313 -0
- posttrain-v1-psi0-all-7task-tune-vlm-direct-dense-actionhead-M.sh +313 -0
- posttrain-v1-psi0-all-7task-tune-vlm-direct-dense-actionhead-perfect-resume.sh +259 -0
- posttrain-v1-psi0-all-7task-tune-vlm-direct-dense-full.sh +427 -0
- posttrain-v1-psi0-all-7task-tune-vlm-direct-dense.sh +305 -0
- posttrain-v1-psi0-all-tune-vlm-direct-dense.sh +297 -0
- posttrain-v1-psi0-cache-complete-smoke.sh +447 -0
- prebuild-multimodal-records-cache.sh +306 -0
- prepare_audio_command_hidden.sh +43 -0
- prepare_full_duplex_continuous_audio.sh +326 -0
- pyproject.toml +85 -0
- uv.lock +0 -0
.env.sample
ADDED
|
@@ -0,0 +1,16 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
HF_TOKEN=
|
| 2 |
+
WANDB_API_KEY=
|
| 3 |
+
WANDB_ENTITY=
|
| 4 |
+
PSI_HOME=/hfm
|
| 5 |
+
DATA_HOME=/hfm/data
|
| 6 |
+
HF_HOME=/hfm/cache
|
| 7 |
+
TORCH_HOME=/hfm/cache
|
| 8 |
+
UV_CACHE_DIR=/hfm/cache
|
| 9 |
+
HF_LEROBOT_HOME=/hfm/data/lerobot
|
| 10 |
+
OMP_NUM_THREADS=8
|
| 11 |
+
TOKENIZERS_PARALLELISM=false
|
| 12 |
+
DEEPSPEED_LOG_LEVEL=warning
|
| 13 |
+
CUDA_LAUNCH_BLOCKING=true
|
| 14 |
+
TF_CPP_MIN_LOG_LEVEL=3
|
| 15 |
+
AV_LOG_LEVEL=quiet
|
| 16 |
+
PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION=python
|
.gitignore
ADDED
|
@@ -0,0 +1,71 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Python-generated files
|
| 2 |
+
__pycache__/
|
| 3 |
+
*.py[oc]
|
| 4 |
+
.pytest_cache/
|
| 5 |
+
.mypy_cache/
|
| 6 |
+
.ruff_cache/
|
| 7 |
+
build/
|
| 8 |
+
dist/
|
| 9 |
+
wheels/
|
| 10 |
+
*.egg-info
|
| 11 |
+
|
| 12 |
+
# Virtual environments
|
| 13 |
+
.venv*
|
| 14 |
+
|
| 15 |
+
/data/
|
| 16 |
+
cache
|
| 17 |
+
*.mp4
|
| 18 |
+
ml-egodex_dataset/ # soft link to the egodex dataset
|
| 19 |
+
wandb/
|
| 20 |
+
runs/
|
| 21 |
+
checkpoints/
|
| 22 |
+
log/
|
| 23 |
+
logs/
|
| 24 |
+
profile_output/
|
| 25 |
+
recovery/
|
| 26 |
+
runtime_indexes/
|
| 27 |
+
playgrounds/
|
| 28 |
+
TODO.md
|
| 29 |
+
*.zip
|
| 30 |
+
*.log
|
| 31 |
+
*.safetensors
|
| 32 |
+
*.ckpt
|
| 33 |
+
|
| 34 |
+
runs_old/
|
| 35 |
+
tmp_dir/
|
| 36 |
+
/tmp/
|
| 37 |
+
.runs*
|
| 38 |
+
|
| 39 |
+
|
| 40 |
+
.env
|
| 41 |
+
.env.local
|
| 42 |
+
|
| 43 |
+
_lerobot_build
|
| 44 |
+
.cache
|
| 45 |
+
.data
|
| 46 |
+
.logs
|
| 47 |
+
|
| 48 |
+
*.local.yaml
|
| 49 |
+
|
| 50 |
+
|
| 51 |
+
vq/
|
| 52 |
+
extern/vq_bet_official/
|
| 53 |
+
src/fast/egodex-*
|
| 54 |
+
*.pkl
|
| 55 |
+
!src/fast/egodex-rel-50w-1x48-v2048-s100
|
| 56 |
+
|
| 57 |
+
.deepspeed_env
|
| 58 |
+
*.bak
|
| 59 |
+
#uv.lock
|
| 60 |
+
.vscode/
|
| 61 |
+
.direnv/
|
| 62 |
+
third_party/
|
| 63 |
+
src/gr00t/.venv/
|
| 64 |
+
props/
|
| 65 |
+
textures/
|
| 66 |
+
test.py
|
| 67 |
+
*.npy
|
| 68 |
+
|
| 69 |
+
unitree_sdk2_python
|
| 70 |
+
|
| 71 |
+
dataset/
|
.gitmodules
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
[submodule "third_party/SIMPLE"]
|
| 2 |
+
path = third_party/SIMPLE
|
| 3 |
+
url = git@github.com:songlin/SIMPLE.git
|
26072112-reuse6-task3131.md
ADDED
|
@@ -0,0 +1,62 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# reuse=6 快速诊断分支记录
|
| 2 |
+
|
| 3 |
+
时间:2026-07-21 12:00(Asia/Shanghai)
|
| 4 |
+
|
| 5 |
+
## 一、可恢复点
|
| 6 |
+
|
| 7 |
+
- 修改前的 reuse=1/无放回工作树:分支 `v26-io4-length-bucket`,commit `fb5a7bea4d2709219bd96c5a4dc9dc409c04a1a2`。
|
| 8 |
+
- 本分支起点:`eeb7339`。它与 run `posttrain.sg.flow1000.cwml.lr1.0e-04.b512.gpus8.2607190305` 使用相同训练代码;从该点恢复了旧的 `reuse_passes=6` 快速链路。
|
| 9 |
+
- 本分支:`v26-26072112-reuse6-task3131-fast`。
|
| 10 |
+
|
| 11 |
+
## 二、WandB 原始对照
|
| 12 |
+
|
| 13 |
+
以下数值由 2026-07-21 直接读取项目 `yth040313-zhejiang-university/psi` 的历史记录得到。
|
| 14 |
+
|
| 15 |
+
| run | eval 数 | 指标 | 第一次 | 最好值(step) | 最新/最终值 | 相对最好值回升 |
|
| 16 |
+
|---|---:|---|---:|---:|---:|---:|
|
| 17 |
+
| v18 纯视频 `uczqi8j1` | 500 | eval/loss | 5.0924 | 0.5894(488k) | 0.6152(500k) | +4.4% |
|
| 18 |
+
| v26 reuse=6 `1xx176cb` | 697 | eval/loss | 5.9274 | 2.3833(274k) | 2.9430(697k) | +23.5% |
|
| 19 |
+
| v26 reuse=6 `1xx176cb` | 697 | text val | 4.8667 | 1.2462(658k) | 1.2717 | +2.0% |
|
| 20 |
+
| v26 reuse=6 `1xx176cb` | 697 | audio val | 7.0652 | 3.6123(132k) | 4.7976 | +32.8% |
|
| 21 |
+
| v26 reuse=6 `1xx176cb` | 697 | audio-command val | 5.2137 | 1.5055(363k) | 1.6895 | +12.2% |
|
| 22 |
+
| v26 reuse=6 `1xx176cb` | 697 | video-clip val | 6.5640 | 2.8787(215k) | 4.0131 | +39.4% |
|
| 23 |
+
| v26 reuse=1 `0ttsbxhe` | 74 | eval/loss | 5.9347 | 2.5237(74k) | 2.5237(74k) | 0.0% |
|
| 24 |
+
|
| 25 |
+
reuse=6 run 的训练 loss 没有同步变坏。以每个目标 step 前后约 5k 步的采样中位数计算:
|
| 26 |
+
|
| 27 |
+
| step | text train | audio train | audio-command train | video train | 总 train |
|
| 28 |
+
|---:|---:|---:|---:|---:|---:|
|
| 29 |
+
| 132k | 1.3517 | 3.2110 | 1.2854 | 2.6019 | 2.1194 |
|
| 30 |
+
| 215k | 1.2093 | 2.9825 | 1.1906 | 2.3810 | 1.9405 |
|
| 31 |
+
| 274k | 1.1784 | 2.7817 | 1.1423 | 2.2772 | 1.8522 |
|
| 32 |
+
| 500k | 1.0516 | 2.3646 | 1.0093 | 1.8978 | 1.5872 |
|
| 33 |
+
| 697k | 0.9701 | 2.1598 | 0.8888 | 1.7643 | 1.4440 |
|
| 34 |
+
|
| 35 |
+
也就是说,audio/video 的 train loss 一直下降,但严格 source 隔离的 validation loss 明显上升,符合过拟合或训练/验证 source 域差异被逐渐放大的形态。因此本次把 audio/video 的全局梯度占比各从 25% 降到 12.5%,用快速 run 验证“模态权重过大”这一假设。
|
| 36 |
+
|
| 37 |
+
## 三、为什么 v18 不能直接证明多 epoch 不会过拟合
|
| 38 |
+
|
| 39 |
+
- v18 的视频划分在 `seed_video_motion_dataset.py::_select_ids` 中直接对稠密 clip record 行做随机切分。同一原视频切出的相邻窗口可能分别进入 train 和 val;当前 v26 则按完整 motion source 隔离,验证更严格,二者的 eval 不能横向等价比较。
|
| 40 |
+
- v18 只训练 `video_clip_text=0.5` 和 `video_clip_only=0.5`,没有当前四模态之间的梯度竞争,也没有当前新增的完整 2264 视频来源。
|
| 41 |
+
- v18 会训练 VLM,当前 cached-hidden v26 不加载也不更新 VLM。
|
| 42 |
+
- v18 在 500k 时学习率已由余弦计划降到约 `5e-8`;它还启用了梯度裁剪和异常 step 保护。本分支按要求不迁入这些做法。
|
| 43 |
+
- v18 从全数据随机采样;v26 为了 OSS 吞吐长时间驻留同一 shard,局部样本相关性更强。
|
| 44 |
+
|
| 45 |
+
因此 v18 提供的真正启发是:当前泛化问题不应只归因于“epoch 数”,还要分别验证 source 隔离、模态权重、固定学习率以及 shard 局部相关性。本分支只改变其中的模态权重,并把 reuse=6 作为提速手段,避免一次实验同时改太多变量。
|
| 46 |
+
|
| 47 |
+
## 四、本分支训练配置
|
| 48 |
+
|
| 49 |
+
- `reuse_passes=6`:恢复与 `2607190305` 相同的快速复用方式,仅用于诊断。
|
| 50 |
+
- 8 卡固定分配:text 3 卡、audio 1 卡、audio-command 3 卡、video-clip 1 卡。
|
| 51 |
+
- 每个 rank 默认 2 个 DataLoader workers。
|
| 52 |
+
- 直连 OSS:全节点 2 个下载槽,每个文件 `parallel=32`,下载完成后驻留 worker RAM,临时文件立即删除。
|
| 53 |
+
- foreach 梯度范数只优化监控速度,不裁剪也不改变梯度。
|
| 54 |
+
- tqdm 每步只输出一行。
|
| 55 |
+
|
| 56 |
+
## 五、验证
|
| 57 |
+
|
| 58 |
+
- `bash -n` 启动脚本通过。
|
| 59 |
+
- 修改文件 Python 编译通过,`git diff --check` 通过。
|
| 60 |
+
- 数据、采样、source split、distributed RNG 和性能相关 CPU 回归测试:35 项通过。
|
| 61 |
+
- 新增测试确认 8 rank 每一步严格是 3/1/3/1,直连 OSS 路径映射正确,scratch 会清理,并发请求同一 shard 只下载一次,foreach 梯度范数与旧算法一致且不修改梯度。
|
| 62 |
+
- 使用真实 `ossutil` 配置对映射后的 text shard 执行只读 `stat` 成功,确认 OSS URI、权限和认证有效;没有启动新的 GPU 训练,也没有干扰正在运行的训练。
|
2608121521-conclusion.md
ADDED
|
@@ -0,0 +1,237 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# HFD 项目阶段性结论(2608121521)
|
| 2 |
+
|
| 3 |
+
本文以当前默认分支上的正式语义版本 19(v19)为主线,并用最近完整跑完的八卡实验
|
| 4 |
+
`runs/posttrain_full_duplex/hfd.gpus8.2608110555` 的实际配置和统计交叉核对。历史实验会明确标成“已回滚”或“仅消融”,不会混入当前正式架构。
|
| 5 |
+
|
| 6 |
+
## 1. CHANGELOG 全分支核对结果
|
| 7 |
+
|
| 8 |
+
### 1.1 审计范围与结论
|
| 9 |
+
|
| 10 |
+
- 已拉取并核对 `66` 个本地分支和 `66` 个 `origin` 远端引用,共 `132` 个 branch refs。
|
| 11 |
+
- 审计前当前 `CHANGELOG.md` 有 `83` 个二级条目;所有同名条目与历史分支中的正文一致,没有发现“标题相同但内容被改写/截断”的情况。
|
| 12 |
+
- 所有历史分支的标题并集比当前文件多 `11` 条。其中 `8` 条属于 HFD 的真实开发/实验历史,已补录回 `CHANGELOG.md`;其余 `3` 条来自曾被错误导入 HFD 的 Psi0-v26 文件树,不能当成 HFD 变更,故明确排除。
|
| 13 |
+
- 这 8 条补录不都代表当前正式功能:有些是关键修复的来源,有些是对照实验,有些后来因违背全双工目标而回滚。补录时均标注了生命周期。
|
| 14 |
+
|
| 15 |
+
### 1.2 补录的 8 条 HFD 历史
|
| 16 |
+
|
| 17 |
+
| 日期 | 历史条目 | 来源分支 | 当前状态 |
|
| 18 |
+
|---|---|---|---|
|
| 19 |
+
| 2026-07-27 | 只读 motion 窗口时间并行提升吞吐 | `2607270014-hfd-throughput` | **已回滚**。它在 motion 阶段冻结语义 KV,与“环境声和新指令随时可能出现”的全双工原则冲突。 |
|
| 20 |
+
| 2026-07-31 | 四角色流式历史记忆 | `2607310204-hfd-history-memory` | **对照实验**。当前正式版不用自定义四角色 KV,而用 Qwen 原生逐层 KV。 |
|
| 21 |
+
| 2026-07-31 | 正式 batch 降为每卡 32 | `2607310204-hfd-history-memory` | **历史配置**。针对旧“完整 source 一次反传”的 OOM;当前逐 clip 更新后正式值已是每卡 128。 |
|
| 22 |
+
| 2026-07-31 | HFDv3 串联 control/action | `origin/2607310158-hfdv3` | **核心思想已吸收**。当前仍是“先预测 token,再把预测 token 送给动作头”,但采用 hard token,未保留早期 straight-through 默认。 |
|
| 23 |
+
| 2026-07-31 | HFDv3 退出清理与显存心跳 | `origin/2607310158-hfdv3` | 历史工程加固记录。 |
|
| 24 |
+
| 2026-07-31 | RTC 首帧参考状态修复 | `2607311949-fix-rtc-reference-hfd` | **已进入正式主线**。当前状态是 `t-1`,RTC 动作首帧 `t-10` 的 delta 必须从 `t-11` 编码。 |
|
| 25 |
+
| 2026-07-31 | RTC 修复并隔离 control/action 梯度 | `origin/2607311949-fix-rtc-control-hfdv3` | **已进入正式主线**。hard 离散 token 下,action loss 不跨 argmax 训练 control logits。 |
|
| 26 |
+
| 2026-08-09 | Future progress 四卡消融 | `2608091607-ablate-future-progress` | **临时消融、已撤回**。正式版继续保留 future progress。 |
|
| 27 |
+
|
| 28 |
+
### 1.3 明确排除的 3 条错误 v26 记录
|
| 29 |
+
|
| 30 |
+
以下标题只存在于错误快照分支 `v26-2608080330-newdata-train-ready`,对应的是 Psi0-v26 数据工程,不属于 HFD:
|
| 31 |
+
|
| 32 |
+
- `2026-08-06 — 严格 stride-1 新视频数据流式接入`
|
| 33 |
+
- `2026-08-07 — 修复 Qwen hidden 双线程预取竞态`
|
| 34 |
+
- `2026-08-08 — 完成 0806 dense hidden 发布并接入正式训练`
|
| 35 |
+
|
| 36 |
+
不把它们合入 HFD 是有意的数据边界,而不是遗漏。
|
| 37 |
+
|
| 38 |
+
## 2. 当前数据量
|
| 39 |
+
|
| 40 |
+
### 2.1 动作和指令资产
|
| 41 |
+
|
| 42 |
+
| 项目 | 数量 | 含义 |
|
| 43 |
+
|---|---:|---|
|
| 44 |
+
| Seed motion metadata | 142,220 条 | 原始动作记录;motion 根目录来自 `Psi0/dataset/seed_all`,metadata 为 `seed_metadata_v004.csv`。 |
|
| 45 |
+
| 动作统计帧 | 51,773,138 行 action;51,915,358 行 state | 用于 39D TextOp 动作/状态统计。 |
|
| 46 |
+
| 去重短指令 | 5,220 条 | 每条有中英文语义文本。 |
|
| 47 |
+
| TTS 音频资产 | 5,220 × 8 = 41,760 个 WAV | 8 个中英文音色;全量文本配对审计通过。 |
|
| 48 |
+
| 连续背景 assignment | 41,694 条 | 已绑定固定办公室背景位置的有效语音 source。 |
|
| 49 |
+
| 训练运行时可用 WAV | 41,656 条 | 对 41,694 条再过滤 38 个异常时长音频后的数量。 |
|
| 50 |
+
| 办公室背景 | 10 条连续 session、8,960 个窗口 | 同一训练 session 内只使用一条连续背景,不循环拼接。 |
|
| 51 |
+
| 混合音频窗口 | 离线合同 388,374;运行时 386,364 | 0.6 秒窗口、0.4 秒步长;运行时数字已扣除异常音频。 |
|
| 52 |
+
| idle / listen 动作池 | 68 / 123 条 | 从 metadata 描述中筛出的正常直立、原地、无倒立/爬行等脏姿态素材。 |
|
| 53 |
+
|
| 54 |
+
### 2.2 正式 v19 的 train/validation 规模
|
| 55 |
+
|
| 56 |
+
先以完整 motion primitive 做稳定哈希切分,再做多轮拼接,因此同一动作的任何窗口都不会同时落入 train 和 validation。
|
| 57 |
+
|
| 58 |
+
| 阶段 | primitive 数 | 每 epoch 使用次数 | 拼接后的 session 数 |
|
| 59 |
+
|---|---:|---:|---:|
|
| 60 |
+
| 原始 motion | 142,220 | — | — |
|
| 61 |
+
| 过滤危险/不适合轮次的动作 | -31,374 | — | — |
|
| 62 |
+
| 可用总数 | 110,846 | — | — |
|
| 63 |
+
| train | 99,457 | 198,914(每条用两个不同音色各一次) | 66,241 条 2~4 轮 session |
|
| 64 |
+
| validation | 11,389 | 11,389(确定性单 variant) | 3,790 条 2~4 轮 session |
|
| 65 |
+
|
| 66 |
+
train/validation 的 primitive source 交集为 `0`。同一 train primitive 在一个 epoch 内不是遍历全部 8 个音色,而是进入两个不同 session、使用两个不同音色和不同邻居组合;这样在可控计算量下兼顾音色覆盖和组合多样性。一个 session 从头到尾固定同一个说话人/语言,避免轮间无意义换人。
|
| 67 |
+
|
| 68 |
+
### 2.3 一个 epoch 到底是多少 step
|
| 69 |
+
|
| 70 |
+
最近完成的八卡 run 使用每卡 `128` 条并行 session,全局最多 `1024` 条。由于当前训练是“每个对齐动作 clip 更新一次参数”,epoch 不能用 `66,241 / 1024` 简单计算:每批 session 会沿时间继续产生大量 stride-1 动作位置,直到该组 session 结束。
|
| 71 |
+
|
| 72 |
+
- 实际日志:`149,516 optimizer steps / epoch`。
|
| 73 |
+
- 已完成的 `500,000` step 约为 `3.34 epoch`。
|
| 74 |
+
- 当前工作树中的启动脚本存在用户尚未提交的 `1,000,000` step、每 `5,000` step 保存/验证设置;如果保持同一数据、八卡和 batch,它约为 `6.69 epoch`。最近已完成 run 本身仍是 `500,000` step、每 `10,000` step 保存/验证,二者不要混为一谈。
|
| 75 |
+
|
| 76 |
+
## 3. 当前数据设计
|
| 77 |
+
|
| 78 |
+
### 3.1 一条 session 的高层结构
|
| 79 |
+
|
| 80 |
+
正式数据不是固定的单轮 `idle → listen → motion`,而是随机拼成 2~4 轮连续交互,例如:
|
| 81 |
+
|
| 82 |
+
`idle → listen₁ → motion₁ → short idle → listen₂ → motion₂ → listen₃ → motion₃ → terminal idle`
|
| 83 |
+
|
| 84 |
+
- 初始 idle 为 `0~7` 个 0.4 秒语义步,即 `0~2.8 秒`。
|
| 85 |
+
- 轮间 idle 采用混合分布:50% 立即进入下一轮,35% 为 0.4~1.2 秒,15% 为 1.6~4.0 秒。
|
| 86 |
+
- 当前版本不造“动作中途打断”case;下一条指令尽量等上一动作完成后再出现。
|
| 87 |
+
- listen 动作按干净 TTS 的真实起声位置延迟,并额外加入 1 个语义步(0.4 秒)反应时间,避免语音还没开口机器人就提前抬手。这里不在推理时引入额外 VAD 分支。
|
| 88 |
+
- listen、idle 和 task motion 的控制标签按**每一帧**构造;跨状态的 30 帧 chunk 可以前半是 listen、后半是 motion,不再把整块粗暴标成同一阶段。控制头仍在每个决策时刻输出一个标量 token。
|
| 89 |
+
|
| 90 |
+
### 3.2 音频不是拼 hidden,而是连续波形在线编码
|
| 91 |
+
|
| 92 |
+
1. 每条 session 选择一段足够长的安静办公室背景,并保持同一个背景源、连续时间位置不变。
|
| 93 |
+
2. idle 时输入纯背景;有人说话时把 TTS 人声直接叠加到同一段背景波形;说完后继续读取紧邻的背景。
|
| 94 |
+
3. 相邻 0.6 秒窗口以 0.4 秒滑动,共享的 0.2 秒重叠区逐采样连续,不会在 idle/listen/motion 边界换另一段噪声。
|
| 95 |
+
4. 混合后的 16 kHz 原始波形在训练和推理时都在线送入同一 Qwen2.5-Omni,不把“语音 hidden”和“背景 hidden”事后相加。
|
| 96 |
+
|
| 97 |
+
**重要巧思:背景声本身也是语义历史的一部分。** 语音结束后仍持续把环境声写入同一 Qwen KV;不能把 motion 阶段理解成“只读旧指令、不再更新语义”。KV 只在完整 session/source 切换时清空,这才符合随时可能出现新指令的全双工场景。
|
| 98 |
+
|
| 99 |
+
### 3.3 动作拼接与坐标处理
|
| 100 |
+
|
| 101 |
+
- 原始 motion 为 29 个关节角、root 位置和四元数。训练表示转换为 39D:`29 joint absolute + 2 local root XY delta + 5 root rotation delta + 1 root height + 2 foot contact`。
|
| 102 |
+
- idle/listen 采用真实自然动作,不再复制固定站立帧。它们保留素材自身合理的 root 高度,避免“把一种腿部姿态套上另一条动作高度”造成脚扎地。
|
| 103 |
+
- 每个新动作先以机器人上一段动作的终点为锚:完整 XY 轨迹和根四元数做同一个刚性 yaw/平移变换。这样改变的是世界坐标系,不改变动作自身的前后/左右语义、相对转向、关节、根高度和路径长度。
|
| 104 |
+
- stationary 素材也先消除录制时的任意世界朝向,再对齐当前机器人朝向;否则一条面朝南录制的 listen 会让当前面朝北的机器人突然转近 180°。
|
| 105 |
+
- 动作间使用 50 帧(1 秒)过渡;stationary 内部使用 20 帧(0.4 秒)平滑。z 高度独立使用素材自身正常高度,不随任务 root 的任意首帧高度一起旋转/硬套。
|
| 106 |
+
|
| 107 |
+
**重点改动:朝向对齐必须旋转“整条根轨迹 + 全部根四元数”,而不是只改第一帧;足端高度必须保留 stationary 自身的物理一致性,而不是加渲染侧防穿地阈值。** 这两项都是从数据根源修复,不是视觉补丁。
|
| 108 |
+
|
| 109 |
+
### 3.4 双速率窗口与 RTC
|
| 110 |
+
|
| 111 |
+
- 动作频率:50 Hz。
|
| 112 |
+
- Qwen 音频窗口:0.6 秒,每 20 动作帧(0.4 秒)更新一次。
|
| 113 |
+
- 动作训练位置:每 1 帧(0.02 秒)滑动一次;同一个 Qwen condition 可供相邻 20 个动作位置复用。
|
| 114 |
+
- 每个动作块覆盖 30 帧:10 帧 RTC 已知前缀 + 20 帧新动作。source 首块没有跨 source 前缀;后续块沿用上一块的 10 帧尾部。
|
| 115 |
+
- v19 在 `idle/listen/motion` token 切换时也保留这 10 帧,只有 source 开头才重置,因此状态切换和普通 chunk 切换遵循同一条连续性规则。
|
| 116 |
+
- 当时同口径 v11 数据审计中,stride-1 动作窗口由 6,867,674 增至 107,761,302,约 `15.69×`。当前多轮/过滤规则后来又变化,所以这个数字表示 stride-1 的增量规模,不冒充当前 v19 的精确窗口总数。
|
| 117 |
+
|
| 118 |
+
**重要巧思:动作以 50 Hz 做密集状态覆盖,昂贵的 Qwen 仍只按部署需要的 2.5 Hz 更新。** 这让模型见到更多初始 state/RTC 组合,又避免把 Qwen 计算放大 20 倍。
|
| 119 |
+
|
| 120 |
+
## 4. 当前模型架构
|
| 121 |
+
|
| 122 |
+
### 4.1 主干数据流与张量形状
|
| 123 |
+
|
| 124 |
+
以下 `B` 为当前有效 batch,`S_a` 为一个 Qwen 音频窗口产生的有效 token 数。
|
| 125 |
+
|
| 126 |
+
| 模块 | 输入 | 输出/作用 |
|
| 127 |
+
|---|---|---|
|
| 128 |
+
| 连续音频 | `B × 9,600` waveform(0.6 s × 16 kHz) | 相邻窗口每 0.4 秒推进。 |
|
| 129 |
+
| Qwen2.5-Omni-3B audio + thinker | waveform + 当前 session 原生 `DynamicCache` | 当前窗口 hidden:`B × S_a × 2048`;当前 token 可注意完整历史 KV。 |
|
| 130 |
+
| Qwen 训练范围 | — | audio tower、embedding、低层 decoder 冻结;只训练最后 2 层 decoder 和 final norm。 |
|
| 131 |
+
| 历史摘要 | 当前窗口最后一个有效、已被历史条件化的 token | `B × 2048`。用于 control 和时长预测。 |
|
| 132 |
+
| control head | `B × 2048` | `LayerNorm → Linear(2048,512) → GELU → Linear(512,3)`,输出 `</listen>`、`</motion>`、`</idle>` logits。 |
|
| 133 |
+
| 串联 control token | 模型自身 hard argmax | 查表 `3 × 2048` embedding,得到 `B × 1 × 2048`;与当前 Qwen hidden 拼接。动作头从不读取 GT token。 |
|
| 134 |
+
| motion-duration head | 同一 `B × 2048` 因果摘要 | `LayerNorm → Linear(2048,512) → GELU → Linear(512,1)`,预测 `log1p(seconds)`。 |
|
| 135 |
+
| 动作条件 | Qwen tokens + predicted control token + 当前 `39D` state + RTC prefix | 全部投影到动作隐空间 1536。 |
|
| 136 |
+
| future progress | `B × 30 × 5` | 两层 MLP 投影到 1536,逐帧加到 RTC prefix 和待生成动作 token;5D 为进度、剩余量及周期 sin/cos 等相位特征。 |
|
| 137 |
+
| Flow Action Transformer | 首块最多 `B × 30 × 39` noisy action;常规块为 `B × 20 × 39` noisy postfix + `B × 10 × 39` clean prefix | 6 个 joint-attention block、hidden 1536、24 heads、head dim 64,约 500,103,359 参数。 |
|
| 138 |
+
| 输出 | action token | 预测 flow velocity;推理用 10-step FlowMatch Euler 从噪声积分出新的 39D 动作。 |
|
| 139 |
+
|
| 140 |
+
补充:`run_config.json` 仍保留一个历史字段 `dim_feedforward=2048`,但当前 `ActionTransformerModel` 构造没有消费这个字段,实际 block 使用 Diffusers `FeedForward` 默认扩展。画结构图时不要错误标成“FFN=2048”。这不造成训推不一致,但属于后续可清理的配置歧义。
|
| 141 |
+
|
| 142 |
+
### 4.2 三个并行任务头的关系
|
| 143 |
+
|
| 144 |
+
1. **控制头**回答“现在应该 idle、listen 还是 motion”。训练和推理都使用模型自己的 hard 预测去条件化动作头。
|
| 145 |
+
2. **时长头**只在每轮 `listen → motion` 边界估计该条 primitive task motion 的总秒数;不包含 idle/listen,也不负责停止动作。推理把首次预测的时长在该轮冻结,只用于构造连续 phase。
|
| 146 |
+
3. **动作头**结合音频语义、历史 KV、hard control、当前 state、RTC 和 phase 生成动作。
|
| 147 |
+
|
| 148 |
+
`action loss` 在 hard argmax 模式下会训练动作 Transformer 和三个 control embedding,但**不会穿过离散 argmax 去训练 control logits**;control head 由三分类 CE 单独训练。这是有意的梯度隔离,避免动作误差反过来扭曲特殊 token 判断。
|
| 149 |
+
|
| 150 |
+
### 4.3 “最新指令文本”只读辅助分支
|
| 151 |
+
|
| 152 |
+
多轮实验发现,完整长 KV 偶尔会让旧指令压过最新指令。因此训练在每轮 `listen → motion` 边界,从动作生成实际使用的同一份 Qwen KV 分出一个临时只读副本,teacher-force 解码 metadata 中与 TTS 完全匹配的当前指令原文:
|
| 153 |
+
|
| 154 |
+
- 文本 CE 只更新已经开放的 Qwen 尾两层和 final norm;LM head 冻结。
|
| 155 |
+
- 查询和答案不会写回生产 KV,不污染下一窗口历史。
|
| 156 |
+
- 推理不运行文本解码,也没有额外“文本中间层”延迟。
|
| 157 |
+
- 它的作用不是把系统改成 ASR→动作串联,而是给 Qwen 一个明确的辅助梯度:“当前有效意图是最新这一条”。
|
| 158 |
+
|
| 159 |
+
**这是比较有巧思的一点:用语言监督增强在线语义记忆,但部署路径仍然是端到端音频→动作。**
|
| 160 |
+
|
| 161 |
+
## 5. 损失函数
|
| 162 |
+
|
| 163 |
+
正式总目标由以下部分组成:
|
| 164 |
+
|
| 165 |
+
- Flow action loss:只监督未知 postfix 和有效帧。39D 分组权重为 joint `1`、root local XY `3`、root rotation `10`、root height `15`、foot contact `0.1`。
|
| 166 |
+
- Control CE:权重 `1.0`。
|
| 167 |
+
- 跨窗口平滑:关节速度误差 `0.10`、关节加速度误差 `0.05`、root 连续性误差 `0.10`。
|
| 168 |
+
- Stationary root-height:`1.0`。
|
| 169 |
+
- Motion duration Smooth-L1:`0.20`。
|
| 170 |
+
- Latest-instruction text CE:`0.10`。
|
| 171 |
+
|
| 172 |
+
平滑 loss 不是把速度/加速度压到零,而是比较“预测的帧间变化”和“GT 的帧间变化”;它包含上一真实历��到新 chunk 第一帧的 seam,因此不会以牺牲走路、挥手等真实动态为代价换取僵硬。
|
| 173 |
+
|
| 174 |
+
## 6. 训练策略
|
| 175 |
+
|
| 176 |
+
### 6.1 当前正式默认
|
| 177 |
+
|
| 178 |
+
- 8 卡时每卡 batch 128,全局最多 1024 条并行 session。
|
| 179 |
+
- 每个对齐动作 clip 都执行一次 loss、反向、八卡梯度平均和 optimizer step;不是等整个 source 结束才反向。
|
| 180 |
+
- BF16;Action LR `1e-4`,Qwen 尾层 LR `2e-6`;AdamW;全局梯度裁剪 `1.0`。
|
| 181 |
+
- cosine scheduler 的最低学习率为各参数组峰值的 5%,即默认 Action 不低于 `5e-6`、Qwen 不低于 `1e-7`。
|
| 182 |
+
- Qwen 数值 KV 在完整 session 内持续保留;`qwen_kv_gradient_horizon_windows=1` 只截断旧窗口的反向计算图,不删除历史 K/V 数值。
|
| 183 |
+
- 长度分桶让同一组各 rank 的 session 长度尽量接近;ragged 尾部 padding 不写 KV、不计 loss,但各 rank 保持同构 collective,避免 DDP 死锁。
|
| 184 |
+
- checkpoint 只在合法 source 边界保存,并可完整恢复模型、optimizer、scheduler、sampler 游标、各 rank RNG、flow RNG 和原 WandB run。
|
| 185 |
+
|
| 186 |
+
### 6.2 当前仍然存在的有意训推差异
|
| 187 |
+
|
| 188 |
+
| 项目 | 训练 | 推理 | 判断 |
|
| 189 |
+
|---|---|---|---|
|
| 190 |
+
| 音频/Qwen/KV | 原始混合波形;0.4 s 更新;整 session KV | 相同 | 已对齐。 |
|
| 191 |
+
| Control | 模型自身 hard token | 模型自身 hard token | 已对齐,无 GT token teacher forcing。 |
|
| 192 |
+
| RTC | 30/20/10;跨状态保留 10 帧 | 相同 | v19 已对齐。 |
|
| 193 |
+
| Action 采样位置 | stride 1 帧 | 每 20 帧执行一次 | 有意的数据增强;不是完全相同的调用频率。 |
|
| 194 |
+
| Qwen 条件 | 每 20 个 dense action step 刷新一次,其余复用 | 每 0.4 秒刷新一次 | 语义刷新频率对齐。 |
|
| 195 |
+
| RTC/state 历史 | 主要来自 GT 连续轨迹 | 来自上一窗口真实生成结果 | 仍有 exposure gap;此前生成历史课程实验因不稳定已回滚。 |
|
| 196 |
+
| Future progress 时长 | 使用 GT primitive motion 时长 | 使用时长头预测,并在该轮冻结 | 当前最明确的剩余条件差异。 |
|
| 197 |
+
| 最新指令文本 | 训练边界有只读辅助 CE | 不解码文本 | 有意的 training-only auxiliary。 |
|
| 198 |
+
|
| 199 |
+
因此,“训练和推理完全逐项相同”并不准确;正确说法是:关键在线语义、hard control、KV 生命周期和 RTC 几何已经对齐,同时保留了两项有意差异——dense state augmentation 和 training-only 辅助监督,并仍有 GT 历史/预测时长造成的闭环 gap。
|
| 200 |
+
|
| 201 |
+
## 7. Eval 与自动推理
|
| 202 |
+
|
| 203 |
+
- validation 与 train 按 primitive source 严格隔离,固定 seed、确定性取样。
|
| 204 |
+
- 同步 eval 只在部署实际使用的 0.4 秒 Qwen 节点上评估,并使用 10 帧 RTC;它仍以 GT state/历史为主,是 teacher-forced 局部定量指标,不等于完整闭环视频质量。
|
| 205 |
+
- 自动推理才是闭环验证:10-step flow,使用上一窗口生成 state/RTC,完整 session 不清 Qwen KV。
|
| 206 |
+
- watcher 当前支持两套固定场景:训练集三轮(抓取→挥手→向左走)和未见 validation 四轮(慢跑→拿取→跳跃→idle),分别输出到独立目录。
|
| 207 |
+
- 每个 run/checkpoint 都保存数据、Qwen、prompt、KV、RTC、control、时长和辅助分支合同;不匹配时明确拒绝加载。watcher 只消费写完 marker 的不可变 `ckpt_<step>`,避免保存/推理竞态。
|
| 208 |
+
|
| 209 |
+
## 8. 最值得在论文/PPT 中重点强调的设计
|
| 210 |
+
|
| 211 |
+
1. **真正持续的全双工语义记忆**:背景、语音、动作执行期都持续更新同一 Qwen KV,只有 session 边界清空。
|
| 212 |
+
2. **双速率训练**:动作 stride-1 提供密集 state/RTC 覆盖,Qwen 仍按 0.4 秒更新,把语义计算与控制频率解耦。
|
| 213 |
+
3. **连续波形级环境建模**:同一办公室背景贯穿整条 session,人声在波形层叠加,再统一经过 Qwen;没有 hidden 拼接突变。
|
| 214 |
+
4. **模型预测 token 串联动作**:先做 turn-taking 决策,再以自身 hard token 条件化动作,训练推理都不喂 GT control。
|
| 215 |
+
5. **多轮、同说话人、不断 KV**:2~4 轮组合训练最新意图与历史依赖;同一 motion 用不同音色和不同邻居复用。
|
| 216 |
+
6. **只读最新指令辅助监督**:直接在真实 action condition 的 KV 上增强“最新意图”辨别,推理仍保持端到端无文本解码。
|
| 217 |
+
7. **时长只负责 phase,不负责 stop**:把“动作处于哪个阶段”和“是否继续 motion”拆成 future progress 与 control 两个问题。
|
| 218 |
+
8. **从数据根部保证几何连续**:整轨迹刚性朝向对齐、stationary 原生 z、GT 导数型平滑 loss、跨状态 RTC 继承共同解决漂移、扎地和切换突变。
|
| 219 |
+
9. **可证明的运行合同**:数据哈希、Qwen 指纹、prompt、解冻层、KV horizon、RTC、control 和 semantics 都写入 checkpoint,防止旧权重被静默错误推理/续训。
|
| 220 |
+
|
| 221 |
+
## 9. 可直接交给 GPT 画图/PPT 的 Prompt
|
| 222 |
+
|
| 223 |
+
### Prompt A:数据构造总图
|
| 224 |
+
|
| 225 |
+
> 请绘制一张 16:9、论文级矢量风格的“Humanoid Full-Duplex 多轮训练数据构造”流程图,白底、深蓝/青绿/橙色点缀,中文标签,信息密度高但整洁。左侧画三类原始资产:① Seed 全身动作库 142,220 条 motion metadata,过滤 31,374 条危险或不适合轮次的动作后保留 110,846 条;② 5,220 条中英文短指令、8 个音色、41,760 个 TTS WAV;③ 10 条连续安静办公室背景、8,960 个 0.6 秒窗口。中间画稳定 source-level train/validation 切分:train 99,457 primitive、validation 11,389、交集 0;train 每个 primitive 用两个不同音色进入两个不同邻居组合,validation 单一确定 variant。再画 2~4 轮 session 拼接时间轴:随机 0~2.8 秒 idle,listen,完整 motion,随机轮间 idle,再进入下一轮,整条 session 固定同一说话人和同一连续背景。突出波形级混合:背景从头到尾连续,人声只是在对应时间段叠加,相邻 0.6 秒窗口以 0.4 秒滑动并共享 0.2 秒,禁止画成 hidden 相加或多段噪声拼接。动作侧画自然 idle 池 68 条、listen 池 123 条,task motion 对齐上一段终点的 XY/yaw,整条根轨迹和四元数一起刚性旋转,stationary 保留自身 root z,动作间用 1 秒平滑过渡。右侧画输出:16 kHz raw waveform 流、50 Hz 39D TextOp 动作流、逐帧 idle/listen/motion 标签、5D future progress、每轮一次 duration target 和 latest-instruction text target。用醒目 callout 标出三点创新:“连续背景波形”“多轮不断 KV”“动作 stride-1 / Qwen stride-20 双速率”。不要加入 VAD 推理模块,不要画离线 hidden 作为当前主输入。
|
| 226 |
+
|
| 227 |
+
### Prompt B:模型架构总图
|
| 228 |
+
|
| 229 |
+
> 请绘制一张适合论文方法章节和答辩 PPT 的 HFD v19 模型架构图,16:9 横向、白底、现代扁平矢量风格,中文主标签并保留关键英文术语。输入是每 0.4 秒到达的 0.6 秒、16 kHz 音频窗口(9,600 samples)和当前机器人 39D state。音频进入 Qwen2.5-Omni-3B audio tower + thinker,输出当前窗口 `S_a × 2048` hidden;画出跨整个 2~4 轮 session 循环回写的原生 Dynamic KV cache,背景、语音和动作执行期都更新,只有 session boundary reset。标出 Qwen audio tower/低层冻结,只训练最后 2 个 decoder layers 和 final norm。用最后一个历史条件化 token `2048D` 分出两个头:control head `2048→512→3` 预测 hard `</idle> / </listen> / </motion>`;duration head `2048→512→1` 只在 listen→motion 边界预测整条 task motion 秒数。将模型自身 hard control 通过 `3×2048` embedding 变成一个 token,与当前 Qwen hidden 串联,明确标注“no GT control token”。另一条训练专用虚线分支从同一 KV 的只读副本进入 frozen LM head,做 latest-instruction teacher-forced CE;标注“training only, no KV pollution, no inference text decoding”。下半部分画 Flow Action Transformer:Qwen/control tokens、当前 `1×39` state、RTC `10×39` clean prefix、`20×39` noisy postfix、`30×5` future progress 和 flow timestep 一起进入 6-block、hidden 1536、24-head joint-attention Transformer,输出 `20×39` flow velocity;推理使用 10-step Euler solver。右侧展开 39D 动作:29 joint angles、2 local root XY delta、5 root rotation delta、1 root height、2 foot contacts。用红色小字标明:action loss 不穿过 hard argmax 训练 control logits,control head 由 CE 训练;duration 只控制 phase,不控制 stop。
|
| 230 |
+
|
| 231 |
+
### Prompt C:训推时序对齐图
|
| 232 |
+
|
| 233 |
+
> 请画一张上下双泳道的“训练 vs 实时推理时序”图,横轴为时间。上方训练:50 Hz motion,每 1 帧滑动一个 30 帧 action target;每 20 个动作位置才输入一次新的 0.6 秒音频窗口并更新 Qwen KV,其余位置复用同一 Qwen condition。下方推理:每 0.4 秒输入一个新音频窗口,动作块采用 30/20/10 RTC,即已知上一块 10 帧尾部、生成 20 帧新动作、继续保留 10 帧供下一块使用;idle/listen/motion 状态切换时也保留 RTC,只有 session 开头清空。用彩色时间段画出 `idle → listen → motion → listen → motion → idle`,每一帧有自己的控制标签,边界 chunk 可以包含两种颜色。画出同一 Qwen KV 横跨所有状态和多个 turn 的连续箭头。旁边列出有意差异:训练 action stride=1、推理 stride=20;训练 future progress 用 GT 时长、推理用冻结的预测时长;训练主要用 GT state/RTC、推理用生成历史。不要把 motion 阶段画成冻结 KV,不要在 token 切换处重置 RTC。
|
| 234 |
+
|
| 235 |
+
### Prompt D:整套汇报 PPT
|
| 236 |
+
|
| 237 |
+
> 请基于以下主题生成一套 10 页中文研究汇报 PPT:《Humanoid Full-Duplex:从连续语音语义到多轮全身动作生成》。视觉风格简洁、学术、适合机器人/多模态会议答辩;每页只保留一个中心结论,少段落、多结构图。第1页问题与目标:机器人在持续环境声中边听边决定 idle/listen/motion;第2页数据规模和严格 source split;第3页连续办公室背景与波形级 TTS 混合;第4页 2~4 轮 session、固定说话人和逐帧控制标签���第5页 Qwen 原生跨窗 KV 与 hard 串联控制;第6页 30/20/10 RTC 和动作 stride-1/Qwen stride-20 双速率;第7页 39D Flow Action Transformer、future progress 与 duration head;第8页最新指令只读文本辅助监督;第9页几何连续性修复,包括整轨迹 heading 对齐、stationary 原生 root z、跨窗口导数 loss 和跨状态 RTC;第10页训练/validation/闭环推理的边界、当前剩余 exposure gap 与下一步。数据必须使用:142,220 原始 motion,110,846 可用 primitive,train 99,457/66,241 sessions,validation 11,389/3,790 sessions,八卡 B128、149,516 step/epoch。重点高亮“持续 KV”“连续波形”“多轮同说话人”“hard token 串联”“只读最新意图监督”“phase 与 stop 解耦”。不要声称 eval loss 等于闭环视频质量,不要把当前系统画成一问一答或 speech 后冻结语义。
|
260904-kv_memory.md
ADDED
|
@@ -0,0 +1,468 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# HFD 可扩展 KV 历史记忆设计说明
|
| 2 |
+
|
| 3 |
+
> 记录日期:2026-09-04
|
| 4 |
+
> 状态说明:当前正式代码已经实现 `latest / previous / first` 三种查询 marker 及最新/历史指令辅助文本 loss;本文进一步说明一种可扩展到任意历史轮次的设计。文中“相对轮次地址”和“软距离 attention 偏置”属于后续方案,不能误写成当前已经完整落地的功能。
|
| 5 |
+
|
| 6 |
+
## 1. 先区分“保存历史”和“读取历史”
|
| 7 |
+
|
| 8 |
+
Qwen 每处理一段新音频,都会在每层 Transformer 中产生 Key 和 Value:
|
| 9 |
+
|
| 10 |
+
```text
|
| 11 |
+
K: [B, H, N, D]
|
| 12 |
+
V: [B, H, N, D]
|
| 13 |
+
```
|
| 14 |
+
|
| 15 |
+
其中:
|
| 16 |
+
|
| 17 |
+
- `B`:batch size;
|
| 18 |
+
- `H`:attention head 数量;
|
| 19 |
+
- `N`:目前累计的历史 token 数量;
|
| 20 |
+
- `D`:每个 attention head 的维度;
|
| 21 |
+
- 所有 head 合起来的隐藏维度为 `H × D = 2048`。
|
| 22 |
+
|
| 23 |
+
假设一条 session 有四轮指令,KV 中保存的时间线大致是:
|
| 24 |
+
|
| 25 |
+
```text
|
| 26 |
+
[初始背景]
|
| 27 |
+
[第1轮语音]
|
| 28 |
+
[动作1期间背景]
|
| 29 |
+
[第2轮语音]
|
| 30 |
+
[动作2期间背景]
|
| 31 |
+
[第3轮语音]
|
| 32 |
+
[动作3期间背景]
|
| 33 |
+
[第4轮语音]
|
| 34 |
+
```
|
| 35 |
+
|
| 36 |
+
新内容会持续追加到 KV 后面。旧内容不会因为 `latest_instruction_loss` 被覆盖。只有以下情况才会真正丢失旧 KV:
|
| 37 |
+
|
| 38 |
+
- source/session 切换时主动 reset;
|
| 39 |
+
- 超过模型最大上下文后被裁剪;
|
| 40 |
+
- 代码显式清空对应 cache 槽。
|
| 41 |
+
|
| 42 |
+
因此:
|
| 43 |
+
|
| 44 |
+
```text
|
| 45 |
+
KV Cache 负责“把历史存下来”;
|
| 46 |
+
Attention 和查询机制负责“这次从历史里读什么”。
|
| 47 |
+
```
|
| 48 |
+
|
| 49 |
+
历史仍然保存在 KV 中,并不等于模型一定能正确读出它。
|
| 50 |
+
|
| 51 |
+
## 2. 为什么只保留完整 KV 仍然可能读错指令
|
| 52 |
+
|
| 53 |
+
假设用户刚说完“向左走”,但 KV 中同时保存着:
|
| 54 |
+
|
| 55 |
+
```text
|
| 56 |
+
第1轮:向前走
|
| 57 |
+
第2轮:抓取
|
| 58 |
+
第3轮:挥手
|
| 59 |
+
第4轮:向左走
|
| 60 |
+
```
|
| 61 |
+
|
| 62 |
+
普通 attention 的分数为:
|
| 63 |
+
|
| 64 |
+
\[
|
| 65 |
+
A_{ij}=\frac{Q_iK_j^T}{\sqrt D}
|
| 66 |
+
\]
|
| 67 |
+
|
| 68 |
+
当前 Query 会根据它与所有历史 Key 的相似度决定关注哪里。如果“向前走”形成的历史特征更强,模型就可能在第4轮结束后错误地重新读取第1轮。
|
| 69 |
+
|
| 70 |
+
此前三轮语音实验正是发现了这个现象:第一轮指令仍然可以被解码出来,甚至比最新指令更容易读取。它说明历史没有被擦除,实际问题是模型没有稳定学会“现在应该读取哪一轮”。
|
| 71 |
+
|
| 72 |
+
## 3. 当前三个 marker 的真实含义和局限
|
| 73 |
+
|
| 74 |
+
当前实现中的三个可学习 marker 是:
|
| 75 |
+
|
| 76 |
+
```text
|
| 77 |
+
latest
|
| 78 |
+
previous
|
| 79 |
+
first
|
| 80 |
+
```
|
| 81 |
+
|
| 82 |
+
它们不是三个记忆槽,也不负责存放三条指令。完整历史依然全部存放在 Qwen KV 中。marker 只是附加在文本查询前面的“查询类型提示”:
|
| 83 |
+
|
| 84 |
+
```text
|
| 85 |
+
latest → 请读取最新一轮
|
| 86 |
+
previous → 请读取最新一轮之前的那一轮
|
| 87 |
+
first → 请读取第一轮
|
| 88 |
+
```
|
| 89 |
+
|
| 90 |
+
例如一条 session 有七轮指令:
|
| 91 |
+
|
| 92 |
+
```text
|
| 93 |
+
KV: [第1轮][第2轮][第3轮][第4轮][第5轮][第6轮][第7轮]
|
| 94 |
+
```
|
| 95 |
+
|
| 96 |
+
此时:
|
| 97 |
+
|
| 98 |
+
```text
|
| 99 |
+
latest → 第7轮
|
| 100 |
+
previous → 第6轮
|
| 101 |
+
first → 第1轮
|
| 102 |
+
```
|
| 103 |
+
|
| 104 |
+
第2~5轮仍然保存在 KV 中,但当前辅助任务没有明确教模型如何点名读取其中任意一轮。因此三个 marker 只是一项“历史寻址是否可学”的小型验证,不是最终的长期记忆架构。
|
| 105 |
+
|
| 106 |
+
## 4. 可扩展方案:为所有历史轮次建立地址
|
| 107 |
+
|
| 108 |
+
### 4.1 为每轮指令记录 turn id
|
| 109 |
+
|
| 110 |
+
数据构造时,为每一轮指令分配递增编号:
|
| 111 |
+
|
| 112 |
+
```text
|
| 113 |
+
第1轮:turn_id = 1
|
| 114 |
+
第2轮:turn_id = 2
|
| 115 |
+
第3轮:turn_id = 3
|
| 116 |
+
...
|
| 117 |
+
```
|
| 118 |
+
|
| 119 |
+
KV 之外,再维护一份与历史 token 对齐的轮次编号:
|
| 120 |
+
|
| 121 |
+
```text
|
| 122 |
+
KV token: [背景][第1轮语音][背景][第2轮语音][背景][第3轮语音]
|
| 123 |
+
turn_id: [ 0 ][11111111][ 1 ][22222222][ 2 ][33333333]
|
| 124 |
+
```
|
| 125 |
+
|
| 126 |
+
`turn_id` 只是地址元数据,不替代 KV,也不包含指令内容。真实语音语义仍然保存在每层的 K/V 中。
|
| 127 |
+
|
| 128 |
+
当当前位于第 `T` 轮时,某段历史距离现在的轮数为:
|
| 129 |
+
|
| 130 |
+
\[
|
| 131 |
+
d=T-\operatorname{turn\_id}
|
| 132 |
+
\]
|
| 133 |
+
|
| 134 |
+
例如当前是第5轮:
|
| 135 |
+
|
| 136 |
+
| 历史轮次 | 相对距离 `d` |
|
| 137 |
+
|---|---:|
|
| 138 |
+
| 第5轮 | 0 |
|
| 139 |
+
| 第4轮 | 1 |
|
| 140 |
+
| 第3轮 | 2 |
|
| 141 |
+
| 第2轮 | 3 |
|
| 142 |
+
| 第1轮 | 4 |
|
| 143 |
+
|
| 144 |
+
这样,每一个中间轮次都有明确地址,不再局限于 `latest / previous / first`。
|
| 145 |
+
|
| 146 |
+
### 4.2 使用“软距离偏置”默认优先读取近期内容
|
| 147 |
+
|
| 148 |
+
可以在 attention 分数上增加一个较弱的、可学习的距离偏置:
|
| 149 |
+
|
| 150 |
+
\[
|
| 151 |
+
A_{ij}
|
| 152 |
+
=
|
| 153 |
+
\frac{Q_iK_j^T}{\sqrt D}
|
| 154 |
+
-
|
| 155 |
+
\alpha_h\log(1+d_j)
|
| 156 |
+
\]
|
| 157 |
+
|
| 158 |
+
其中:
|
| 159 |
+
|
| 160 |
+
- `d_j`:第 `j` 个历史 token 距离当前有几轮;
|
| 161 |
+
- `α_h`:第 `h` 个 attention head 学到的衰减强度;
|
| 162 |
+
- `log(1+d)`:距离越远,扣分逐渐增加,但永远不会直接变成不可见。
|
| 163 |
+
|
| 164 |
+
例如原始语义匹配分数是:
|
| 165 |
+
|
| 166 |
+
```text
|
| 167 |
+
第1轮:8.0
|
| 168 |
+
第2轮:3.0
|
| 169 |
+
第3轮:4.0
|
| 170 |
+
第4轮:5.0
|
| 171 |
+
```
|
| 172 |
+
|
| 173 |
+
加入轻微距离偏置后,可能变为:
|
| 174 |
+
|
| 175 |
+
```text
|
| 176 |
+
第1轮:6.8
|
| 177 |
+
第2轮:2.2
|
| 178 |
+
第3轮:3.6
|
| 179 |
+
第4轮:5.0
|
| 180 |
+
```
|
| 181 |
+
|
| 182 |
+
默认情况下,第4轮更容易被选择;但如果第1轮与当前问题的语义相关性足够高,它依然可能胜出���旧历史没有被删除。
|
| 183 |
+
|
| 184 |
+
这与硬 attention mask 有本质区别:
|
| 185 |
+
|
| 186 |
+
```text
|
| 187 |
+
硬 mask:把远处历史设为 -∞,模型完全看不到;
|
| 188 |
+
软偏置:只降低默认关注度,需要时仍能重新关注。
|
| 189 |
+
```
|
| 190 |
+
|
| 191 |
+
理想情况下,不同 attention head 可以学出不同的记忆职责:
|
| 192 |
+
|
| 193 |
+
- 一部分 head 的 `α` 较大,主要负责最近指令;
|
| 194 |
+
- 一部分 head 的 `α` 接近 0,负责较长历史;
|
| 195 |
+
- 其余 head 主要依靠语义相关性在多轮之间检索。
|
| 196 |
+
|
| 197 |
+
这种设计既提供近期优先级,又不强迫所有 attention head 遗忘长期内容。
|
| 198 |
+
|
| 199 |
+
## 5. 可扩展的相对轮次查询 embedding
|
| 200 |
+
|
| 201 |
+
如果需要明确查询某一条历史,可使用相对距离 `r`:
|
| 202 |
+
|
| 203 |
+
```text
|
| 204 |
+
r = 0:最新一轮
|
| 205 |
+
r = 1:上一轮
|
| 206 |
+
r = 2:上两轮
|
| 207 |
+
r = 3:上三轮
|
| 208 |
+
...
|
| 209 |
+
```
|
| 210 |
+
|
| 211 |
+
不再只准备三个固定向量,而是用一个小函数把任意距离转换成 2048 维查询向量:
|
| 212 |
+
|
| 213 |
+
\[
|
| 214 |
+
e_r=\operatorname{MLP}(\log(1+r))
|
| 215 |
+
\]
|
| 216 |
+
|
| 217 |
+
形状为:
|
| 218 |
+
|
| 219 |
+
```text
|
| 220 |
+
r: [B, 1]
|
| 221 |
+
↓ MLP
|
| 222 |
+
e_r: [B, 1, 2048]
|
| 223 |
+
```
|
| 224 |
+
|
| 225 |
+
然后把它放在正常自然语言查询前面:
|
| 226 |
+
|
| 227 |
+
```text
|
| 228 |
+
[e_r][Transcribe the requested instruction]
|
| 229 |
+
```
|
| 230 |
+
|
| 231 |
+
这个向量只表达“请找距离当前第几轮”,不包含正确指令答案。假设当前是第7轮:
|
| 232 |
+
|
| 233 |
+
```text
|
| 234 |
+
r=0 → 第7轮
|
| 235 |
+
r=1 → 第6轮
|
| 236 |
+
r=4 → 第3轮
|
| 237 |
+
r=6 → 第1轮
|
| 238 |
+
```
|
| 239 |
+
|
| 240 |
+
因此它可以覆盖所有中间历史,不受三个固定角色限制。
|
| 241 |
+
|
| 242 |
+
## 6. 训练 loss 如何构造
|
| 243 |
+
|
| 244 |
+
假设当前已经完成第5轮,对应真实指令文本为:
|
| 245 |
+
|
| 246 |
+
```text
|
| 247 |
+
Y1, Y2, Y3, Y4, Y5
|
| 248 |
+
```
|
| 249 |
+
|
| 250 |
+
### 6.1 最新指令 loss
|
| 251 |
+
|
| 252 |
+
每次指令结束,都要求模型从完整 KV 中解码当前指令:
|
| 253 |
+
|
| 254 |
+
\[
|
| 255 |
+
L_{\text{latest}}
|
| 256 |
+
=
|
| 257 |
+
-\frac{1}{|Y_5|}
|
| 258 |
+
\sum_l
|
| 259 |
+
\log P(Y_{5,l}\mid KV,e_0,Y_{5,<l})
|
| 260 |
+
\]
|
| 261 |
+
|
| 262 |
+
对应的人话流程是:
|
| 263 |
+
|
| 264 |
+
```text
|
| 265 |
+
给模型完整的五轮KV;
|
| 266 |
+
通过e_0告诉它“读取最新一轮”;
|
| 267 |
+
让它逐token还原第5轮真实指令;
|
| 268 |
+
预测错误就产生loss。
|
| 269 |
+
```
|
| 270 |
+
|
| 271 |
+
每句话按自身 token 数量取平均,防止长句天然拥有更大的训练权重。
|
| 272 |
+
|
| 273 |
+
### 6.2 任意历史指令 loss
|
| 274 |
+
|
| 275 |
+
再随机抽取一条旧指令。例如抽到第2轮:
|
| 276 |
+
|
| 277 |
+
```text
|
| 278 |
+
当前轮 T = 5
|
| 279 |
+
目标轮 i = 2
|
| 280 |
+
相对距离 r = T - i = 3
|
| 281 |
+
```
|
| 282 |
+
|
| 283 |
+
对应 loss 为:
|
| 284 |
+
|
| 285 |
+
\[
|
| 286 |
+
L_{\text{history}}
|
| 287 |
+
=
|
| 288 |
+
-\frac{1}{|Y_2|}
|
| 289 |
+
\sum_l
|
| 290 |
+
\log P(Y_{2,l}\mid KV,e_3,Y_{2,<l})
|
| 291 |
+
\]
|
| 292 |
+
|
| 293 |
+
如果模型错误地解码成第1轮、第3轮或者第5轮,都会产生损失。下次训练可能抽到另一轮;经过大量 epoch 后,中间所有历史轮次都能得到监督。
|
| 294 |
+
|
| 295 |
+
### 6.3 历史轮次采样策略
|
| 296 |
+
|
| 297 |
+
没有必要在每个边界把全部历史都解码一遍,否则训练开销会随 session 长度快速增加。更经济的做法是每次额外抽一个历史目标:
|
| 298 |
+
|
| 299 |
+
```text
|
| 300 |
+
50%:选择上一轮;
|
| 301 |
+
50%:从其余所有更早轮次中均匀抽取一轮。
|
| 302 |
+
```
|
| 303 |
+
|
| 304 |
+
还应按照相对距离分桶统计:
|
| 305 |
+
|
| 306 |
+
```text
|
| 307 |
+
距离1
|
| 308 |
+
距离2
|
| 309 |
+
距离3~4
|
| 310 |
+
距离5~8
|
| 311 |
+
距离9+
|
| 312 |
+
```
|
| 313 |
+
|
| 314 |
+
训练和验证都要观察每个距离桶,防止总体准确率很好,但模型实际上只会读取最近一两轮。
|
| 315 |
+
|
| 316 |
+
### 6.4 总训练目标
|
| 317 |
+
|
| 318 |
+
历史记忆 loss 不替代原来的动作目标,而是作为较小的辅助监督加入:
|
| 319 |
+
|
| 320 |
+
\[
|
| 321 |
+
L
|
| 322 |
+
=
|
| 323 |
+
L_{\text{action}}
|
| 324 |
+
+\lambda_cL_{\text{control}}
|
| 325 |
+
+\lambda_dL_{\text{duration}}
|
| 326 |
+
+\lambda_sL_{\text{smooth}}
|
| 327 |
+
+\lambda_lL_{\text{latest}}
|
| 328 |
+
+\lambda_hL_{\text{history}}
|
| 329 |
+
\]
|
| 330 |
+
|
| 331 |
+
可沿用当前较保守的初始权重:
|
| 332 |
+
|
| 333 |
+
```text
|
| 334 |
+
lambda_latest = 0.10
|
| 335 |
+
lambda_history = 0.05
|
| 336 |
+
```
|
| 337 |
+
|
| 338 |
+
实际权重还要结合各分支梯度大小验证,不能只看 raw loss 数值。辅助文本任务的目标是整理 Qwen 的语义与寻址能力,不能压过动作生成主任务。
|
| 339 |
+
|
| 340 |
+
需要特别区分两种“距离权重”:
|
| 341 |
+
|
| 342 |
+
- 在线 attention 可以对远处历史施加轻微默认降权;
|
| 343 |
+
- 历史训练 loss 不能随着距离迅速变小,否则模型虽然保存了旧 KV,却始终没学会如何读取远处历史。
|
| 344 |
+
|
| 345 |
+
## 7. 一条三轮 session 的完整训练过程
|
| 346 |
+
|
| 347 |
+
假设训练数据是:
|
| 348 |
+
|
| 349 |
+
```text
|
| 350 |
+
idle
|
| 351 |
+
→ 指令1:向前走
|
| 352 |
+
→ motion1
|
| 353 |
+
→ 指令2:挥手
|
| 354 |
+
→ motion2
|
| 355 |
+
→ 指令3:抓取
|
| 356 |
+
→ motion3
|
| 357 |
+
```
|
| 358 |
+
|
| 359 |
+
训练流程为:
|
| 360 |
+
|
| 361 |
+
1. 音频按照正式节奏每 `0.4 s` 更新一次 Qwen;
|
| 362 |
+
2. 新 K/V 追加到同一个 session cache;
|
| 363 |
+
3. 整个 session 内不 reset;
|
| 364 |
+
4. 指令1结束时,`L_latest` 要求从 KV 解码“向前走”;
|
| 365 |
+
5. 指令2结束时,`L_latest` 解码“挥手”,`L_history` 解码“向前走”;
|
| 366 |
+
6. 指令3结束时,`L_latest` 解码“抓取”,`L_history` 随机要求解码“挥手”或“向前走”;
|
| 367 |
+
7. 文本辅助 loss 与 control/action loss 一起反向传播;
|
| 368 |
+
8. 更新可训练的 Qwen 尾层、memory query embedding 以及其他参与当前目标的模块;
|
| 369 |
+
9. 文本查询使用由生产 KV 建立的一次性 side cache,不把查询和答案写回正式音频 KV。
|
| 370 |
+
|
| 371 |
+
KV 本身是一次 forward 中产生的中间状态,不是直接被 optimizer 更新的参数。loss 真正训练的是:
|
| 372 |
+
|
| 373 |
+
```text
|
| 374 |
+
Qwen以后怎样把不同轮语音编码成容易区分的K/V;
|
| 375 |
+
读取端以后怎样根据相对轮次地址,从完整历史里找到正确内容。
|
| 376 |
+
```
|
| 377 |
+
|
| 378 |
+
如果为了控制显存而 detach 了较早 KV,历史 loss 对很早的编码过程不会进行完整长距离反向传播,此时它主要训练查询端和可训练 Qwen 尾层如何读取已有 KV。这一点需要在正式实现时结合当前 TBPTT/cache detach 逻辑专项验证。
|
| 379 |
+
|
| 380 |
+
## 8. 为什么 latest loss 不会直接擦除历史
|
| 381 |
+
|
| 382 |
+
当前最新指令辅助分支的基本过程是:
|
| 383 |
+
|
| 384 |
+
1. 在真实指令边界复制当前生产 KV;
|
| 385 |
+
2. 在副本上追加“只转写最新指令”的查询;
|
| 386 |
+
3. 使用 metadata/TTS 中的精确文本计算 teacher-forced token CE;
|
| 387 |
+
4. 查询和答案不写回后续正式音频 cache。
|
| 388 |
+
|
| 389 |
+
因此它没有执行以下任何操作:
|
| 390 |
+
|
| 391 |
+
```text
|
| 392 |
+
删除旧K/V;
|
| 393 |
+
覆盖旧K/V;
|
| 394 |
+
只保留最后一轮;
|
| 395 |
+
在每轮结束时reset主cache。
|
| 396 |
+
```
|
| 397 |
+
|
| 398 |
+
但是,“物理上没有删除”不等于“参数训练后一定会主动使用”。共享 Qwen 参数仍有可能学会忽略旧信息。因此必须使用自由文本解码分别询问最新、上一条、第一条以及任意中间轮次,才能验证历史是否真正可检索。
|
| 399 |
+
|
| 400 |
+
此前实验中第一轮的自由解码语义准确率明显高于最新一轮,这直接否定了“历史已经被 latest loss 全部擦掉”的说法;当时真正的问题反而是第一轮过强、最新轮次寻址不稳定。
|
| 401 |
+
|
| 402 |
+
## 9. 推理时是否需要真的生成文本
|
| 403 |
+
|
| 404 |
+
正常动作推理不需要把辅助文本显示出来。文本解码分支主要承担两项工作:
|
| 405 |
+
|
| 406 |
+
```text
|
| 407 |
+
训练:通过精确文本答案约束Qwen语义和历史寻址;
|
| 408 |
+
评估:直接检查KV记住了什么、读取的是不是正确轮次。
|
| 409 |
+
```
|
| 410 |
+
|
| 411 |
+
推理时动作模型仍然直接消费经过这些辅助 loss 训练的 Qwen 语义特征,不必额外运行完整文本 decoder,因此原则上不增加在线动作延迟。
|
| 412 |
+
|
| 413 |
+
但“文本能读对”并不自动等价于“动作一定执行正确”。最终必须同时评估:
|
| 414 |
+
|
| 415 |
+
- 各历史距离桶的自由文本解码准确率;
|
| 416 |
+
- 是否混淆最新指令和旧指令;
|
| 417 |
+
- control 状态切换是否正确;
|
| 418 |
+
- 动作头是否执行了对应语义;
|
| 419 |
+
- 涉及历史引用的闭环指令是否真正完成。
|
| 420 |
+
|
| 421 |
+
当前正式训练中,文本辅助 loss 可以更新可训练的 Qwen 尾层;此前 marker 小实验则冻结 Qwen、只训练 marker,用于单独验证“寻址提示是否可学”。两者不能混为同一个实验结论。
|
| 422 |
+
|
| 423 |
+
## 10. 如何避免训练和推理不一致
|
| 424 |
+
|
| 425 |
+
正式动作路径不能使用只有训练数据才知道的 GT 轮次边界,否则训练时地址完全正确,推理时却无法构造同样的 turn id。
|
| 426 |
+
|
| 427 |
+
安全做法应分两层:
|
| 428 |
+
|
| 429 |
+
1. **辅助文本监督**:可以使用数据构造已知的轮次和精确文本,因为它只负责训练/评估记忆能力,不直接向动作路径泄漏未来答案;
|
| 430 |
+
2. **正式在线 attention**:如果启用轮次距离偏置,轮次推进必须来自推理时也能获得的信号,例如模型自身预测的指令完成边界,而不能只使用 GT 边界。
|
| 431 |
+
|
| 432 |
+
更保守的落地顺序是:
|
| 433 |
+
|
| 434 |
+
1. 保持主 Qwen attention 不变;
|
| 435 |
+
2. 先把三个固定 marker 扩展成任意相对距离 `e(r)`;
|
| 436 |
+
3. 使用随机历史轮次文本解码 loss 训练和验证;
|
| 437 |
+
4. 确认中间历史和远距离历史检索明显改善;
|
| 438 |
+
5. 再考虑把轻微的轮次距离偏置加入主 attention。
|
| 439 |
+
|
| 440 |
+
直接修改 Qwen attention mask 可能影响 Flash Attention、训练速度、显存和 checkpoint 合同,因此不应在没有独立 A/B 的情况下直接并入正式版本。
|
| 441 |
+
|
| 442 |
+
## 11. 该方案仍然存在的边界
|
| 443 |
+
|
| 444 |
+
- 它只能保留 Qwen 最大上下文范围内的全部历史,不是无限记忆;超过上下文后仍需要历史压缩或分层 memory;
|
| 445 |
+
- 默认“距离越远权重越小”只能作为弱先验,不能成为硬规则。例如用户说“重复最开始的动作”时,第一轮必须能够重新获得最高注意力;
|
| 446 |
+
- 辅助文本检索效果必须通过自由解码评估,teacher-forced token accuracy 往往偏乐观;
|
| 447 |
+
- 如果推理时轮次边界判断错误,依赖 turn id 的在线偏置也会跟着错,因此必须单独测量边界漂移;
|
| 448 |
+
- 历史文本可检索只是必要条件,不是动作闭环正确的充分条件。
|
| 449 |
+
|
| 450 |
+
## 12. 核心总结
|
| 451 |
+
|
| 452 |
+
```text
|
| 453 |
+
KV Cache:保存完整的多轮语音历史;
|
| 454 |
+
turn id:给每一轮建立地址;
|
| 455 |
+
相对距离 embedding:告诉读取端要找距离当前第几轮;
|
| 456 |
+
软距离 attention 偏置:默认更关注近期内容,但不屏蔽远处历史;
|
| 457 |
+
latest loss:保证当前指令容易读取;
|
| 458 |
+
随机 history loss:保证上一轮、任意中间轮和远距离历史仍可读取;
|
| 459 |
+
side cache:避免辅助文本查询污染正式流式音频历史。
|
| 460 |
+
```
|
| 461 |
+
|
| 462 |
+
最终目标不是“只记住最新指令”,而是:
|
| 463 |
+
|
| 464 |
+
```text
|
| 465 |
+
默认执行最新意图;
|
| 466 |
+
需要时能够准确读取任意历史轮次;
|
| 467 |
+
旧历史始终保留,关注权重由当前任务和语义决定。
|
| 468 |
+
```
|
260904-lzk交接文档.md
ADDED
|
@@ -0,0 +1,447 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# HFD_lzk 上手与交接文档
|
| 2 |
+
|
| 3 |
+
> 更新时间:2026-09-04
|
| 4 |
+
>
|
| 5 |
+
> 面向读者:第一次接触 HFD、了解基础深度学习/PyTorch 概念的开发者
|
| 6 |
+
>
|
| 7 |
+
> 代码位置:/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/HFD_lzk
|
| 8 |
+
|
| 9 |
+
## 1. 先记住三件事
|
| 10 |
+
|
| 11 |
+
1. **HFD 做什么**:机器人持续接收流式语音,在听的同时判断当前应当 idle、listen 还是 motion,并连续生成 Unitree G1 动作。它不是“一句话输入完,再一次性生成整段动作”的离线模型。
|
| 12 |
+
2. **这是隔离副本**:本目录没有原 HFD 的 Git 历史、remote、SSH key 或凭证;可以本地建分支和 commit,但默认不能 push。不要修改相邻的原仓库 .../code/HFD。
|
| 13 |
+
3. **大数据都在仓库外**:数据集、Qwen 权重和训练结果不在 Git 中。当前副本只含约 45 MB 代码与小型资源,runs/ 是独立空目录。
|
| 14 |
+
|
| 15 |
+
原始快照来自 2026-09-04 的 HFD 当前工作树,原仓库 HEAD 为 a79e7c6。副本创建时主动排除了 .git、runs/、.venv-psi 实体、recovery/、cache、日志、checkpoint、视频、.env 和凭证。
|
| 16 |
+
|
| 17 |
+
## 2. 用一句话理解整个系统
|
| 18 |
+
|
| 19 |
+
每 0.4 秒让 Qwen 更新一次“听到目前为止的语义记忆”,每 0.02 秒让动作模型根据这份语义、机器人当前状态、上一窗口留下的动作前缀和动作全局进度,生成接下来的连续动作。
|
| 20 |
+
|
| 21 |
+
~~~text
|
| 22 |
+
16 kHz waveform
|
| 23 |
+
-> Qwen2.5-Omni streaming audio encoder + native causal KV
|
| 24 |
+
-> semantic hidden [B, S, 2048]
|
| 25 |
+
|-> control head -> [B, 3] -> idle/listen/motion condition token
|
| 26 |
+
|-> duration head -> [B, 48] (only at listen -> motion)
|
| 27 |
+
|-> text readout losses (training-only at instruction boundaries)
|
| 28 |
+
-> Action Transformer
|
| 29 |
+
inputs: semantic tokens + control token + state + RTC prefix
|
| 30 |
+
+ flow timestep + future progress
|
| 31 |
+
output: flow velocity [B, 30, 39]
|
| 32 |
+
-> execute/predict 20 new motion frames, retain 10-frame RTC overlap
|
| 33 |
+
~~~
|
| 34 |
+
|
| 35 |
+
其中 S 是当前 Qwen 音频窗口产生的有效语义 token 数,随输入而变,不是固定常数。
|
| 36 |
+
|
| 37 |
+
## 3. 目录和入口文件
|
| 38 |
+
|
| 39 |
+
| 位置 | 作用 |
|
| 40 |
+
|---|---|
|
| 41 |
+
| posttrain-full-duplex-audio-command.sh | 当前正式训练唯一首选入口,默认语义版本 24 |
|
| 42 |
+
| infer_render_auto_multiturn.sh | 自动监听 checkpoint,运行多任务 train/val 推理与渲染的入口 |
|
| 43 |
+
| infer_render_auto.sh | 自动套件调用的较底层单 case worker |
|
| 44 |
+
| scripts/train.py | 通用训练主程序 |
|
| 45 |
+
| scripts/infer/run_multiturn_infer_suite.py | 多任务、多 GPU 推理任务调度 |
|
| 46 |
+
| scripts/infer/full_duplex_audio_to_motion.py | 流式音频到动作的核心推理实现 |
|
| 47 |
+
| src/psi/data/seed/full_duplex_audio_command_dataset.py | 最关键的数据构造代码;在线拼 session、音频、动作与逐帧标签 |
|
| 48 |
+
| src/psi/models/full_duplex_psi0.py | Qwen 流式 KV、控制头、时长头、文本记忆辅助分支和动作头串接 |
|
| 49 |
+
| src/psi/models/psi0.py | 约 5 亿参数的 Action Transformer 与 flow-matching 动作生成 |
|
| 50 |
+
| src/psi/trainers/posttrain_full_duplex.py | 训练/验证循环、逐 source KV 槽、动态补位、loss 和指标 |
|
| 51 |
+
| tests/test_full_duplex_audio_command.py | 数据与时间轴核心回归 |
|
| 52 |
+
| tests/test_hfdv2_qwen_streaming.py | Qwen streaming/KV/model contract 回归 |
|
| 53 |
+
| tests/test_full_duplex_ddp_sync.py | 多卡同步与优化器语义回归 |
|
| 54 |
+
| tests/test_full_duplex_streaming_infer.py | 推理路径回归 |
|
| 55 |
+
| CHANGELOG.md | 真实演化历史;改代码前先搜索相关功能 |
|
| 56 |
+
| EXPERIMENT.md | 重要实验设计与结论 |
|
| 57 |
+
| 260904-kv_memory.md | 当前/历史指令 KV 记忆的详细解释和后续设计 |
|
| 58 |
+
|
| 59 |
+
建议阅读顺序:本文件 → CHANGELOG.md 最近几版 → dataset 的构造入口 → trainer 的单窗口 forward → model 的 forward_stream。
|
| 60 |
+
|
| 61 |
+
## 4. 外部数据和模型路径
|
| 62 |
+
|
| 63 |
+
这些路径由正式脚本默认读取,**共享且应视为只读**:
|
| 64 |
+
|
| 65 |
+
| 内容 | 当前路径 |
|
| 66 |
+
|---|---|
|
| 67 |
+
| 连续办公室背景与混音语音 | /cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/videogen/hfd_audio_command_continuous_office_v1 |
|
| 68 |
+
| 背景分配表 | 上述目录的 manifests/source_background_assignments.jsonl |
|
| 69 |
+
| 背景 session manifest | 上述目录的 manifests/background_sessions.jsonl |
|
| 70 |
+
| 连续音频完成标记 | 上述目录的 reports/production_complete.json |
|
| 71 |
+
| 原始动作库 | /cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/Psi0/dataset/seed_all |
|
| 72 |
+
| 动作 metadata | 上述目录的 metadata/seed_metadata_v004.csv |
|
| 73 |
+
| 主指令 TTS metadata | /cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/qwen3-tts/metadata.jsonl |
|
| 74 |
+
| Stop 指令语音 | /cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/videogen/hfd_control_commands_v1/manifest.jsonl |
|
| 75 |
+
| Backchannel 语音 | /cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/videogen/hfd_backchannel_commands_v2_short/manifest.jsonl |
|
| 76 |
+
| Hesitation/Pause 语音 | /cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/videogen/hfd_hesitation_pause_v2/manifest.jsonl |
|
| 77 |
+
| Qwen2.5-Omni-3B | /cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/team_workspace/public_ckpt/Qwen/Qwen2.5-Omni-3B |
|
| 78 |
+
|
| 79 |
+
路径都可通过训练脚本中的 HFD_* 环境变量覆盖。不要直接改共享 manifest 或共享 WAV;若要造新数据,使用类似 .../yetinghong/videogen/lzk_<dataset_name> 的新目录,并在自己的命令中显式覆盖路径。
|
| 80 |
+
|
| 81 |
+
项目当前使用**原始波形在线编码**,旧 hidden-state cache 已不属于正式输入,不要重新引入。
|
| 82 |
+
|
| 83 |
+
## 5. 数据是怎么构造出来的
|
| 84 |
+
|
| 85 |
+
### 5.1 原始元素
|
| 86 |
+
|
| 87 |
+
一条训练 session 不是磁盘上预先存好的单个大样本,而是在 __getitem__ 中由以下元素在线合成:
|
| 88 |
+
|
| 89 |
+
- 一到多条 motion primitive:关节、根节点移动/朝向/高度、足端接触;
|
| 90 |
+
- 对应指令的 TTS 波形:中英文、多个性别/音色;
|
| 91 |
+
- 一条连续办公室背景音;
|
| 92 |
+
- idle/listen 动作池及不同状态间的平滑过渡;
|
| 93 |
+
- 逐帧控制标签:</listen>、</motion>、</idle>;
|
| 94 |
+
- future progress、RTC 前缀、任务类型和事件边界等监督信息。
|
| 95 |
+
|
| 96 |
+
同一个 session 通常拼接 2~4 轮,整条 session 固定一个音色、使用连续的同一背景,并共享一条 Qwen KV 历史;只有换 session/source 才清空 KV。
|
| 97 |
+
|
| 98 |
+
### 5.2 六类正式能力
|
| 99 |
+
|
| 100 |
+
| 任务 | 训练目标 |
|
| 101 |
+
|---|---|
|
| 102 |
+
| Turn-taking | 空闲后听完整指令,再执行对应动作,完成后合理回到等待状态 |
|
| 103 |
+
| Interruption | 旧动作未结束时收到新指令,先进入 listen,再平滑切换到新动作 |
|
| 104 |
+
| Emergency Stop | 动作中听到停止命令,进入 idle;没有新有效指令时不能重新触发旧动作 |
|
| 105 |
+
| Backchannel | 动作中听到“嗯、对、uh、yeah”等短反馈,不误判为新指令,原动作继续 |
|
| 106 |
+
| Hesitation / self-repair | 用户停顿、拖长语气词或改口时保持 listen,只执行最终明确意图 |
|
| 107 |
+
| Pause | 一条指令句中出现思考停顿时继续等待,不提前开始动作 |
|
| 108 |
+
|
| 109 |
+
控制标签是**逐动作帧**生成的,不是一个 30 帧 chunk 只能有一种标签。因此一个窗口跨越边界时,前面可以是 idle,后面可以是 listen 或 motion。
|
| 110 |
+
|
| 111 |
+
### 5.3 当前默认规模和分布
|
| 112 |
+
|
| 113 |
+
以下是语义版本 24 在 CHANGELOG.md 中记录的正式构造规模;若修改参数,以新训练启动日志为准:
|
| 114 |
+
|
| 115 |
+
- Train:267,850 条合成 session,约 577,086,270 个 stride-1 窗口;
|
| 116 |
+
- Validation:7,698 条独立 session,约 16,519,542 个窗口;
|
| 117 |
+
- train/val 的 primitive 切分独立,不能把训练动作放进 validation;
|
| 118 |
+
- 按窗口估算的训练占比约为:Turn-taking 47.9%、Interruption 7.4%、Stop 10.4%、Backchannel 13.4%、Hesitation 12.5%、Pause 8.3%;
|
| 119 |
+
- 最长约 0.1% session 在正式训练中默认过滤,暂不把超长上下文拖入当前阶段;
|
| 120 |
+
- 每个 primitive 在训练组合中有两个静态音色变体;单条 session 内不换说话人。
|
| 121 |
+
|
| 122 |
+
这里“session 比例”和“窗口比例”不是一回事:长 session 会贡献更多 0.02 秒窗口,所以最终真正被优化器看到的比例以窗口统计为准。
|
| 123 |
+
|
| 124 |
+
### 5.4 重要的数据质量设计
|
| 125 |
+
|
| 126 |
+
- idle 使用安全站立素材的**相对微小残差**叠加到当前姿态,避免从一个随机站姿插值到另一个站姿时凭空挥手;
|
| 127 |
+
- 静止素材做直立、双脚支撑、关节连续性审计;
|
| 128 |
+
- 状态连接时统一根节点 XY/yaw 坐标,Z 高度单独保持姿态自身的合理值,避免脚穿地;
|
| 129 |
+
- 状态间使用平滑过渡,当前默认 turn transition 为 50 帧;
|
| 130 |
+
- 语音和背景可重叠,但一条 session 内背景连续,不拼接两段突变噪声;
|
| 131 |
+
- hesitation 的显式“改成/I mean”和无提示词直接重说均有覆盖;
|
| 132 |
+
- Stop 插入时考虑动作活跃度和剩余时长,避免等动作已经自然结束才发出 Stop。
|
| 133 |
+
|
| 134 |
+
## 6. 时间轴与张量形状
|
| 135 |
+
|
| 136 |
+
### 6.1 两个时间尺度
|
| 137 |
+
|
| 138 |
+
- 动作帧率:50 FPS,即每帧 0.02 秒;
|
| 139 |
+
- 动作训练 stride:1 帧,每 0.02 秒有一个动作起点;
|
| 140 |
+
- 音频窗口:0.6 秒,即 [B, 9600] 个 16 kHz waveform sample;
|
| 141 |
+
- Qwen 更新 stride:20 动作帧,即每 0.4 秒更新一次语义/KV;
|
| 142 |
+
- 两次 Qwen 更新之间的 19 个动作起点复用最近一次因果语义结果;这与部署时每 0.4 秒处理一块新音频一致。
|
| 143 |
+
|
| 144 |
+
### 6.2 RTC 动作几何
|
| 145 |
+
|
| 146 |
+
~~~text
|
| 147 |
+
noisy/target action: [B, 30, 39]
|
| 148 |
+
RTC known prefix: [B, 10, 39]
|
| 149 |
+
new postfix: [B, 20, 39]
|
| 150 |
+
current state: [B, 1, 39]
|
| 151 |
+
future progress: [B, 30, 5]
|
| 152 |
+
~~~
|
| 153 |
+
|
| 154 |
+
30 帧 chunk 覆盖 0.6 秒;前 10 帧是从上一窗口延续来的已知上下文,后 20 帧是新的 0.4 秒动作。推理时保存已生成动作到内存,下个窗口直接拿它作为 RTC 前缀;换 source 时才使用明确的首帧初始化逻辑。
|
| 155 |
+
|
| 156 |
+
### 6.3 39 ��动作表示
|
| 157 |
+
|
| 158 |
+
模型内部的 TextOp motion 是:
|
| 159 |
+
|
| 160 |
+
~~~text
|
| 161 |
+
[0:29] 29D joint absolute angles
|
| 162 |
+
[29:31] 2D local root XY delta
|
| 163 |
+
[31:36] 5D root rotation delta representation
|
| 164 |
+
[36:37] 1D root height
|
| 165 |
+
[37:39] 2D foot contact
|
| 166 |
+
~~~
|
| 167 |
+
|
| 168 |
+
所有维度按 assets/stats/seed_all_g1_motion39_textop_stats.json 归一化。
|
| 169 |
+
|
| 170 |
+
## 7. 模型架构
|
| 171 |
+
|
| 172 |
+
### 7.1 Qwen 流式语义主干
|
| 173 |
+
|
| 174 |
+
- Backbone:Qwen2.5-Omni-3B 的 audio Thinker;
|
| 175 |
+
- 输入:当前 0.6 秒原始混合波形和该 source 的历史原生 Dynamic KV;
|
| 176 |
+
- 输出:H_t = [B, S, 2048];
|
| 177 |
+
- 每 0.4 秒把新音频追加到各 source 独立 KV;
|
| 178 |
+
- 正式训练仅解冻 Qwen 最后 2 个 decoder layer 和 final norm,其余 Qwen 冻结;Qwen 学习率默认 2e-6。
|
| 179 |
+
|
| 180 |
+
KV 的生命周期是整条多轮 session,因此旧指令不会因为进入 motion 或听到白噪音而被清空。动态 source 补位时,每个 batch 槽有自己的 source ID 和 KV;旧 source 结束后只重置该槽,再放入新 source。
|
| 181 |
+
|
| 182 |
+
### 7.2 Control Head
|
| 183 |
+
|
| 184 |
+
Qwen 的因果语义摘要 [B, 2048] 经过:
|
| 185 |
+
|
| 186 |
+
~~~text
|
| 187 |
+
LayerNorm(2048)
|
| 188 |
+
-> Linear(2048, 512)
|
| 189 |
+
-> GELU
|
| 190 |
+
-> Linear(512, 3)
|
| 191 |
+
-> logits [B, 3]
|
| 192 |
+
~~~
|
| 193 |
+
|
| 194 |
+
三个类别依次是 </listen>、</motion>、</idle>。正式版本不把机器人 state/progress 输入 Control Head,避免标签泄漏;控制判断主要来自语音语义历史。
|
| 195 |
+
|
| 196 |
+
训练和推理都对 logits 做 hard argmax。预测类别再查 Embedding(3, 2048) 得到一个 [B, 1, 2048] 条件 token,接到 H_t 后送给 Action Transformer:
|
| 197 |
+
|
| 198 |
+
~~~text
|
| 199 |
+
action condition: [B, S + 1, 2048]
|
| 200 |
+
~~~
|
| 201 |
+
|
| 202 |
+
hard argmax 保证训推一致,但 action loss 不能穿过离散选择去更新 Control Head;Control Head 主要由 control CE 和 Stop 事件 loss 训练。动作 loss 仍能训练 control embedding 和动作网络。
|
| 203 |
+
|
| 204 |
+
### 7.3 动作时长头与 future progress
|
| 205 |
+
|
| 206 |
+
在真实的 listen -> motion 边界,时长头根据 Qwen 摘要输出:
|
| 207 |
+
|
| 208 |
+
~~~text
|
| 209 |
+
LayerNorm(2048) -> Linear(2048, 512) -> GELU -> Linear(512, 48)
|
| 210 |
+
duration logits: [B, 48]
|
| 211 |
+
~~~
|
| 212 |
+
|
| 213 |
+
48 个 bin 覆盖 0.5~180 秒的 log1p(duration) 范围。训练目标是**当前这一轮 primitive motion 本身**的时长,不包括 idle、listen、其他轮或整条 session;软标签可落在相邻 bin。推理取预测分布中位数作为该轮计划时长。
|
| 214 |
+
|
| 215 |
+
计划时长用来构造 5 维 future progress,让动作头知道当前处于动作的开始、中段还是结束,减少周期动作错相位、提前结束或重复启动。
|
| 216 |
+
|
| 217 |
+
### 7.4 最新指令与历史指令文本辅助监督
|
| 218 |
+
|
| 219 |
+
只靠动作 loss 很难确认长 KV 中“最新一句到底是什么”。因此训练在每个真实指令边界复制一份当前生产 KV,临时提出文本查询,并用正确 TTS 文本做 teacher-forced token CE:
|
| 220 |
+
|
| 221 |
+
- latest loss:读取当前最新指令,权重 0.10;
|
| 222 |
+
- memory loss:非首轮再读取 previous 或 first 指令,权重 0.05;
|
| 223 |
+
- 查询前加入 latest/previous/first 三个可学习 marker,告诉 Qwen 本次要取哪一轮;
|
| 224 |
+
- 三个 marker 是三种**查询方式**,不是只存三条记忆。完整历史仍保存在 Qwen KV 中;当前实现只是还不能任意指定所有中间轮次;
|
| 225 |
+
- 临时查询不会写回生产 KV,因此不会把答案泄漏给后续动作;
|
| 226 |
+
- 正式在线推理不运行文本 decoder,这些分支只是训练时增强语义记忆。
|
| 227 |
+
|
| 228 |
+
更通用的任意历史寻址设计见 260904-kv_memory.md,目前尚未并入正式模型。
|
| 229 |
+
|
| 230 |
+
### 7.5 Action Transformer
|
| 231 |
+
|
| 232 |
+
- 隐藏维度 1536;
|
| 233 |
+
- 6 个 joint action-language transformer blocks;
|
| 234 |
+
- 24 个 attention heads,每头 64 维;
|
| 235 |
+
- 输入包括 Qwen/control 条件、当前 state、30 帧 noisy action、10 帧 RTC 前缀、flow timestep 和 5D future progress;
|
| 236 |
+
- 输出 [B, 30, 39] flow velocity;
|
| 237 |
+
- 训练用 flow matching,推理默认使用 10-step flow solver。
|
| 238 |
+
|
| 239 |
+
## 8. Loss 怎么算
|
| 240 |
+
|
| 241 |
+
总 loss 是以下目标的加权和:
|
| 242 |
+
|
| 243 |
+
1. **Action flow loss**:39D 动作的主监督。维度组权重为 joint=1、root XY=3、root rotation=10、root height=15、foot contact=0.1;
|
| 244 |
+
2. **Control CE**:逐帧/有效语义决策点的 idle/listen/motion 分类,权重 1.0;
|
| 245 |
+
3. **连续性 loss**:joint velocity 0.10、joint acceleration 0.05、跨窗口 root continuity 0.10;
|
| 246 |
+
4. **静止高度 loss**:idle/listen 等静止段 root height 权重 1.0;
|
| 247 |
+
5. **Motion duration distribution loss**:权重 0.20;
|
| 248 |
+
6. **Latest instruction text loss**:权重 0.10;
|
| 249 |
+
7. **Historical instruction memory loss**:权重 0.05;
|
| 250 |
+
8. **Stop 事件 loss**:进入 idle=0.5、Stop 后持续不 motion=1.0、收到下一条真实指令后解除=1.0;每个事件内部归一化,避免静默长短决定总权重。
|
| 251 |
+
|
| 252 |
+
Control 标签虽然逐帧存在,但 Qwen 语义只每 0.4 秒更新一次;中间动作帧复用语义,不重复用完全相同的语义表示做互相冲突的 control 更新。
|
| 253 |
+
|
| 254 |
+
## 9. 训练策略
|
| 255 |
+
|
| 256 |
+
当前正式默认值:
|
| 257 |
+
|
| 258 |
+
| 参数 | 默认值 |
|
| 259 |
+
|---|---:|
|
| 260 |
+
| 语义版本 | 24 |
|
| 261 |
+
| GPU 数 | 从 CUDA_VISIBLE_DEVICES 自动检测,正式���常 8 |
|
| 262 |
+
| 每卡 active source slots | 192 |
|
| 263 |
+
| 总 optimizer steps | 1,250,000 |
|
| 264 |
+
| Action peak LR | 8e-5 |
|
| 265 |
+
| Control LR | Action LR × 0.25 = 2e-5 |
|
| 266 |
+
| Qwen LR | 2e-6 |
|
| 267 |
+
| Warmup | 12,500 steps |
|
| 268 |
+
| Scheduler | cosine,最低为 peak 的 5% |
|
| 269 |
+
| Max grad norm | 1.0 |
|
| 270 |
+
| Checkpoint | 每 5,000 step |
|
| 271 |
+
| Validation | 每 20,000 step,固定 64 个全局 source,本地 B=8 |
|
| 272 |
+
| 精度/并行 | BF16 + DDP |
|
| 273 |
+
| DataLoader workers | 每 rank 16 |
|
| 274 |
+
|
| 275 |
+
训练采用 stride-1 密集动作起点,但每个动作窗口都是普通监督学习/flow matching,不是强化学习。每个 dense window 默认执行一次 optimizer update;同一 source 的 KV 跨窗口保留。
|
| 276 |
+
|
| 277 |
+
为了减少 padding,source 按精确流式长度分桶;短 source 结束后,在合法 Qwen 边界给该 KV 槽动态补入新 source。补位不把不同 source 的历史混在一起。
|
| 278 |
+
|
| 279 |
+
## 10. 第一次运行
|
| 280 |
+
|
| 281 |
+
### 10.1 进入目录和环境
|
| 282 |
+
|
| 283 |
+
~~~bash
|
| 284 |
+
cd /cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/HFD_lzk
|
| 285 |
+
source .venv-psi/bin/activate
|
| 286 |
+
~~~
|
| 287 |
+
|
| 288 |
+
.venv-psi 是指向原 HFD 现有环境的只读复用链接。**不要在里面安装或卸载包。** 如果确实需要改依赖,请单独建环境,然后:
|
| 289 |
+
|
| 290 |
+
~~~bash
|
| 291 |
+
export PSI_VENV=/path/to/lzk-owned-venv
|
| 292 |
+
~~~
|
| 293 |
+
|
| 294 |
+
### 10.2 先做不占 GPU 的配置检查
|
| 295 |
+
|
| 296 |
+
~~~bash
|
| 297 |
+
HFD_CONFIG_ONLY=1 \
|
| 298 |
+
HFD_REPORT_TO=none \
|
| 299 |
+
bash posttrain-full-duplex-audio-command.sh
|
| 300 |
+
~~~
|
| 301 |
+
|
| 302 |
+
它会检查默认配置合同并打印数据、模型、loss 和调度参数,但不会启动 torchrun。
|
| 303 |
+
|
| 304 |
+
### 10.3 正式训练
|
| 305 |
+
|
| 306 |
+
先与项目负责人协调 GPU。训练脚本会暂时停止同一机器上的共享占卡程序,结束后再恢复它。
|
| 307 |
+
|
| 308 |
+
~~~bash
|
| 309 |
+
cd /cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/HFD_lzk
|
| 310 |
+
CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7 \
|
| 311 |
+
bash posttrain-full-duplex-audio-command.sh
|
| 312 |
+
~~~
|
| 313 |
+
|
| 314 |
+
默认输出只会写到:
|
| 315 |
+
|
| 316 |
+
~~~text
|
| 317 |
+
/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/HFD_lzk/runs/
|
| 318 |
+
~~~
|
| 319 |
+
|
| 320 |
+
WandB 默认开启,需要由使用者自己在 shell 中提供 WANDB_API_KEY,或创建不进 Git 的个人 .env。不要从原 HFD 复制凭证文件。
|
| 321 |
+
|
| 322 |
+
### 10.4 小型 smoke test
|
| 323 |
+
|
| 324 |
+
不要把 smoke 参数误当正式参数。一个四卡、20 step 的最小示例:
|
| 325 |
+
|
| 326 |
+
~~~bash
|
| 327 |
+
CUDA_VISIBLE_DEVICES=0,1,2,3 \
|
| 328 |
+
HFD_REPORT_TO=none \
|
| 329 |
+
HFD_TRAIN_BATCH_SIZE=8 \
|
| 330 |
+
HFD_EFFECTIVE_SOURCES_PER_GPU=8 \
|
| 331 |
+
HFD_LIMIT_SOURCES=128 \
|
| 332 |
+
HFD_MAX_TRAINING_STEPS=20 \
|
| 333 |
+
HFD_CHECKPOINTING_STEPS=20 \
|
| 334 |
+
HFD_VALIDATION_STEPS=20 \
|
| 335 |
+
HFD_VAL_NUM_BATCHES=8 \
|
| 336 |
+
bash posttrain-full-duplex-audio-command.sh
|
| 337 |
+
~~~
|
| 338 |
+
|
| 339 |
+
## 11. 推理与渲染
|
| 340 |
+
|
| 341 |
+
给一个由 **HFD_lzk 自己生成**、且包含 run_config.json 的 run 目录:
|
| 342 |
+
|
| 343 |
+
~~~bash
|
| 344 |
+
bash infer_render_auto_multiturn.sh \
|
| 345 |
+
/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/HFD_lzk/runs/<run-name>
|
| 346 |
+
~~~
|
| 347 |
+
|
| 348 |
+
自动套件会监听 immutable ckpt_<step>,并把 turn-taking、interruption、stop、backchannel、hesitation、pause 的 train/val case 分配到空闲 GPU 并行运行。动作核心按训练相同的 0.6 秒音频窗口、0.4 秒 Qwen stride、30/20/10 RTC 和 hard control 执行。
|
| 349 |
+
|
| 350 |
+
**不要直接把原 HFD 的 run 目录传给该脚本**,因为推理会在传入的 run 内新建状态文件、日志和视频。若必须检查原 checkpoint,应先把必要 checkpoint 与 run_config.json 复制到 HFD_lzk 自己的临时 run。
|
| 351 |
+
|
| 352 |
+
## 12. 完美续训
|
| 353 |
+
|
| 354 |
+
完整续训需要同时恢复 model、optimizer、scheduler、sampler/source 边界状态,并沿用原 WandB run ID。基本形式是:
|
| 355 |
+
|
| 356 |
+
~~~bash
|
| 357 |
+
HFD_RESUME_FROM_CHECKPOINT=/absolute/path/to/HFD_lzk/runs/<run>/ckpt_<step> \
|
| 358 |
+
HFD_RESUME_LOAD_FULL_STATE=1 \
|
| 359 |
+
HFD_WANDB_RESUME=must \
|
| 360 |
+
HFD_WANDB_ID=<original-wandb-run-id> \
|
| 361 |
+
bash posttrain-full-duplex-audio-command.sh
|
| 362 |
+
~~~
|
| 363 |
+
|
| 364 |
+
只加载 model.safetensors 不等于完美续训。不要用 owner 的原 run 直接续训;先复制到 LZK 自己的空间。
|
| 365 |
+
|
| 366 |
+
## 13. 修改后的最低检查
|
| 367 |
+
|
| 368 |
+
文档或 shell 修改:
|
| 369 |
+
|
| 370 |
+
~~~bash
|
| 371 |
+
bash -n posttrain-full-duplex-audio-command.sh
|
| 372 |
+
bash -n infer_render_auto_multiturn.sh
|
| 373 |
+
git diff --check
|
| 374 |
+
~~~
|
| 375 |
+
|
| 376 |
+
全双工核心修改至少运行:
|
| 377 |
+
|
| 378 |
+
~~~bash
|
| 379 |
+
pytest -q \
|
| 380 |
+
tests/test_full_duplex_audio_command.py \
|
| 381 |
+
tests/test_hfdv2_qwen_streaming.py \
|
| 382 |
+
tests/test_full_duplex_ddp_sync.py \
|
| 383 |
+
tests/test_full_duplex_streaming_infer.py
|
| 384 |
+
~~~
|
| 385 |
+
|
| 386 |
+
路径迁移还应运行:
|
| 387 |
+
|
| 388 |
+
~~~bash
|
| 389 |
+
pytest -q tests/test_hfd_storage_paths.py
|
| 390 |
+
~~~
|
| 391 |
+
|
| 392 |
+
先跑与改动最直接相关的 case;通过后再扩大测试范围。涉及 DDP、BF16、KV 槽或动态补位的修改,CPU 单测通过后仍需真实多卡 smoke,不能只凭静态阅读宣布可用。
|
| 393 |
+
|
| 394 |
+
## 14. Git 隔离与日常协作
|
| 395 |
+
|
| 396 |
+
当前目录是一个全新的本地 Git 仓库:
|
| 397 |
+
|
| 398 |
+
- 分支:lzk-260904-handoff;
|
| 399 |
+
- git remote -v 应为空;
|
| 400 |
+
- .git/hooks/pre-push 会拒绝任何 push;
|
| 401 |
+
- git config push.default nothing 让无参数 git push 直接失败;
|
| 402 |
+
- local Git identity 只写在本目录 .git/config,不动全���配置。
|
| 403 |
+
|
| 404 |
+
推荐流程:
|
| 405 |
+
|
| 406 |
+
~~~bash
|
| 407 |
+
git switch -c lzk-YYMMDDHHMM-short-topic
|
| 408 |
+
# 修改代码,同时更新 CHANGELOG.md
|
| 409 |
+
git status --short
|
| 410 |
+
git diff --check
|
| 411 |
+
# 运行测试
|
| 412 |
+
git add <明确列出的文件>
|
| 413 |
+
git commit -m "简洁说明修改内容"
|
| 414 |
+
~~~
|
| 415 |
+
|
| 416 |
+
本地 commit 不会进入 owner 的仓库。需要把成果交回时,先给 owner 提供 commit hash 和 diff,由 owner 审核后决定如何人工移植;不要自行添加原 HFD remote。
|
| 417 |
+
|
| 418 |
+
## 15. 最容易踩的坑
|
| 419 |
+
|
| 420 |
+
- **混淆动作 stride 和 Qwen stride**:动作每 1 帧训练一次,Qwen 每 20 帧更新一次;二者不是同一个参数。
|
| 421 |
+
- **错误清空 KV**:只能在 source/session 切换时清空,idle、listen、motion、背景噪声和轮次切换都属于同一连续历史。
|
| 422 |
+
- **把 GT control 喂给动作头**:正式训练和推理都使用模型自己的 hard control,不能训练时偷看 GT。
|
| 423 |
+
- **RTC 使用 GT 而推理使用预测历史**:推理要把上一窗口已生成的后续动作保存在内存,下一次作为 10 帧前缀;source 首帧单独初始化。
|
| 424 |
+
- **状态标签按整块生成**:标签必须逐帧,边界窗口允许混合状态。
|
| 425 |
+
- **误改共享数据**:manifest 的 SHA256 参与训练/推理合同,重写路径或内容会让旧 checkpoint 拒绝加载。
|
| 426 |
+
- **直接对原 run 做推理/续训**:会向原目录写文件,破坏隔离。
|
| 427 |
+
- **只看总 loss**:还要分别看 action、control accuracy、Stop reactivation、pause/hesitation premature-motion、duration 和文本 token accuracy。
|
| 428 |
+
- **把低 GPU 利用率一律归因于模型**:先区分同步 validation、checkpoint、DataLoader 等待和真正模型计算;项目已有相应 speed/eval 指标。
|
| 429 |
+
|
| 430 |
+
## 16. 当前仍值得研究的问题
|
| 431 |
+
|
| 432 |
+
1. hard control 的离散 argmax 阻断 action loss 到 Control Head;已有消融未证明 straight-through 或直接三行词表头更优,所以正式版暂时保留可靠的 MLP + CE;
|
| 433 |
+
2. latest/previous/first marker 能验证近期与历史可读,但还不能任意寻址第 k 个历史指令;后续方案见 260904-kv_memory.md;
|
| 434 |
+
3. Control 边界事件相对稀疏,Stop、Backchannel、Interruption、Hesitation 的平衡仍需看事件级指标,而不能只看逐帧 accuracy;
|
| 435 |
+
4. 时长分布比单点回归更合理,但循环/开放式动作仍不天然拥有唯一结束时刻;
|
| 436 |
+
5. 闭环推理质量会受到动作误差累计、RTC 边界、方向和足端稳定性的共同影响,低 teacher-forced eval loss 不等于视频一定好;
|
| 437 |
+
6. 未来加入新任务时,先定义“它与已有任务的最小语义差别”和事件级成功条件,再设计数据,避免用互相矛盾的标签污染控制边界。
|
| 438 |
+
|
| 439 |
+
## 17. 开始改代码前的检查清单
|
| 440 |
+
|
| 441 |
+
- [ ] 我当前位于 HFD_lzk,不是 HFD;
|
| 442 |
+
- [ ] git remote -v 没有输出;
|
| 443 |
+
- [ ] 我理解这次修改影响 data、Qwen/KV、control、action、loss、trainer 还是 inference 中的哪一层;
|
| 444 |
+
- [ ] 我没有写共享数据目录或 owner 的 run;
|
| 445 |
+
- [ ] 我为边界条件增加了最小回归测试;
|
| 446 |
+
- [ ] 我更新了 CHANGELOG.md;
|
| 447 |
+
- [ ] 我先做 local commit,并把 diff 交给 owner 审核,而不是 push。
|
260925-third-party人声重点版更新报告.md
ADDED
|
@@ -0,0 +1,66 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# third-party 非指令人声重点版
|
| 2 |
+
|
| 3 |
+
> 历史版本 `speech_v3_260925`,数据与 GT 视频保留。当前交付见 [中英文交叉版报告](260926-third-party中英交叉版更新报告.md);本页旧版视频不代表新增素材已完成听审。
|
| 4 |
+
|
| 5 |
+
## 1. 本轮改了什么
|
| 6 |
+
|
| 7 |
+
| 项目 | 结果 |
|
| 8 |
+
|---|---|
|
| 9 |
+
| 数据总量 | **27,584 sessions**:train 26,800/validation 784;约 301.054 小时 |
|
| 10 |
+
| 保持不变 | 原目标指令、目标语音、动作监督、时间线及 session 时长 |
|
| 11 |
+
| 本轮重点 | 中英文非指令人声:突然插话、短时多人声、持续多人背景 |
|
| 12 |
+
| 语义筛选 | 排除筛查出的动作/控制指令、请求句及 VQA 表达;不新增 VQA 任务 |
|
| 13 |
+
| clean/额外环境声/白噪声 | 配额全部归零;**原 source 自带室内底噪保留** |
|
| 14 |
+
| TTS | 本轮使用真实录音,没有额外生成 TTS |
|
| 15 |
+
| 替换方式 | 全部 27,584 条音轨一对一更新;旧版保留回退,不与新版拼接 |
|
| 16 |
+
|
| 17 |
+
## 2. Session 配比
|
| 18 |
+
|
| 19 |
+
| 类型 | train | validation | 合计 | 约占比 |
|
| 20 |
+
|---|---:|---:|---:|---:|
|
| 21 |
+
| 单人短插话 | 10,720 | 314 | 11,034 | 40% |
|
| 22 |
+
| 短时三人声 | 5,360 | 157 | 5,517 | 20% |
|
| 23 |
+
| 持续 2~4 人背景 | 10,720 | 313 | 11,033 | 40% |
|
| 24 |
+
|
| 25 |
+
背景中文、英文各 **13,792 条**,与目标同语种或跨语种均有。难度 easy/medium/hard 约为 50%/35%/15%。
|
| 26 |
+
|
| 27 |
+
## 3. 素材来源与实际使用
|
| 28 |
+
|
| 29 |
+
| 语言与来源 | 素材库:片段/说话人 | 实际使用:片段/说话人 | 原始人声时长 |
|
| 30 |
+
|---|---:|---:|---:|
|
| 31 |
+
| 中文:AliMeeting 真人会议 | 5,533/280 | 5,525/280 | 4.381 小时 |
|
| 32 |
+
| 英文:LibriSpeech 真人朗读 | 2,651/248 | 2,597/245 | 7.178 小时 |
|
| 33 |
+
| **合计** | **8,184/528** | **8,122/525** | **11.559 小时** |
|
| 34 |
+
|
| 35 |
+
两套语料原用于语音识别,AliMeeting 还用于说话人分离;有转写,便于筛除指令内容。说话人数按来源 ID 统计,未做跨语料声纹去重。本轮新增人声的说话人、来源组及音频哈希在 train/validation 间无交叉。
|
| 36 |
+
|
| 37 |
+
相较旧版,素材由 7,205 增至 8,184 条,时长由 7.15 增至 11.56 小时;更严格筛选后说话人由 605 减至 528。**不是所有多样性指标都增加。**候选库保留的 235 个历史环境片段本轮零使用。
|
| 38 |
+
|
| 39 |
+
## 4. 背景放在哪里
|
| 40 |
+
|
| 41 |
+
| 类型 | 放置规则 | 实际与目标语音区间重叠 |
|
| 42 |
+
|---|---|---:|
|
| 43 |
+
| 单人短插话 | 1.5~8 秒完整原片段,在可完整放下的范围内随机起点 | 46.04% |
|
| 44 |
+
| 短时三人声 | 三人完整片段混合,最长 8 秒;起点同样随机 | 55.99% |
|
| 45 |
+
| 持续多人背景 | 5,517 条安排全程;5,516 条随机覆盖 session 的 40%~80% | 99.33% |
|
| 46 |
+
|
| 47 |
+
整体约 **80% 为随机位置的局部人声、20% 为全程背景区间**。随机发生在构造时,不是每个训练 epoch 重抽;不固定“指令前/两轮间/动作期间”配额。两条指令之间也可能仍在执行动作。
|
| 48 |
+
|
| 49 |
+
持续背景按说话人分轨串接完整原片段,不裁词尾;事件内片段复用率约 **1.20%**。“全程”指安排区间,不保证每一刻都有人发声;上表是事件边界重叠,不是 VAD 发声重叠。
|
| 50 |
+
|
| 51 |
+
## 5. 验收与查看入口
|
| 52 |
+
|
| 53 |
+
| 检查/材料 | 结果或入口 |
|
| 54 |
+
|---|---|
|
| 55 |
+
| 全量严格审计 | **通过,0 error**;[严格审计](data/hfd_third_party_speech_v3/production/reports/strict_audit.json) |
|
| 56 |
+
| 新旧一对一核对 | **通过,目标与监督逐条不变**;[替换审计](data/hfd_third_party_speech_v3/production/reports/replacement_audit.json) |
|
| 57 |
+
| 数据/GT 聚焦回归 | 195 项通过 |
|
| 58 |
+
| 快速试听 | [12 组 base/augmented 配对](data/hfd_third_party_speech_v3/production/reports/listening_review.md) |
|
| 59 |
+
| 分层试听 | [240 条完整 session,48 层各 5 条](data/hfd_third_party_speech_v3/production/reports/expanded_listening_review.md) |
|
| 60 |
+
| GT 渲染视频 | [12 条带字幕视频](data/hfd_third_party_speech_v3/gt_review_v1/README.md),共 6 分 29.6 秒;三类各 4 条、中英各 6 条,完整解码校验通过 |
|
| 61 |
+
| 当前版本与冻结哈希 | [hfd_third_party_current.json](hfd_third_party_current.json) |
|
| 62 |
+
| 完整构造参数 | [构造记录](data/hfd_third_party_speech_v3/production/reports/pilot_build.json) |
|
| 63 |
+
|
| 64 |
+
正式目录:`data/hfd_third_party_speech_v3/production`;素材只以 `work/catalog_reviewed` 的 registry 为准,不扫描 raw/normalized 全目录导入。过程音频和原始来源需保留,许可与文件哈希见素材记录。
|
| 65 |
+
|
| 66 |
+
**边界:**视频展示正式音轨与 GT 标签,不是模型预测;本轮全程 CPU、未训练。规则与转写复核不能代替人工听审;混合会议/朗读片段不是真实连贯对话,也不能证明模型已获得泛化提升。人工审核、训练 loader 接入及模型效果验证仍待完成。
|
260926-third-party中英交叉版更新报告.md
ADDED
|
@@ -0,0 +1,53 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# third-party 中英文交叉版更新
|
| 2 |
+
|
| 3 |
+
> `cross_v4_260926` 最初交叉版的构造与验收记录。后续已原位提高 2,714 条背景音量,独立全量验收、完整严格审计及 GT 后验均通过;最新状态见[音量修复表](260927-third-party背景音量修复.md)。未训练,人工试听待审核。
|
| 4 |
+
|
| 5 |
+
## 配比与素材
|
| 6 |
+
|
| 7 |
+
表中素材数量均为本地筛选后的数量,不是原始数据集总量。背景语言配比按 session 计,四舍五入后为中文 60%、英文 40%;每种语言均覆盖对话和朗读。
|
| 8 |
+
|
| 9 |
+
| 背景类别 | 来源 | 可用 / 实际使用片段 | 来源 speaker ID(可用 / 使用) | session | 占比 |
|
| 10 |
+
|---|---|---:|---:|---:|---:|
|
| 11 |
+
| 中文对话 | AliMeeting | 5,533 / 5,482 | 280 / 280 | 8,275 | 30% |
|
| 12 |
+
| 中文朗读 | [AISHELL-1](https://www.openslr.org/33/) | 16,000 / 14,890 | 400 / 400 | 8,275 | 30% |
|
| 13 |
+
| 英文对话 | [AMI](https://groups.inf.ed.ac.uk/ami/corpus/) | 1,843 / 1,803 | 141 / 141 | 5,517 | 20% |
|
| 14 |
+
| 英文朗读 | LibriSpeech | 2,651 / 2,485 | 248 / 245 | 5,517 | 20% |
|
| 15 |
+
| 合计 | 全部真人录音,本次未使用 TTS | **26,027 / 24,660** | **1,069 / 1,066** | **27,584** | **100%** |
|
| 16 |
+
|
| 17 |
+
新增两库原用于中文语音识别、英文会议语音研究;取带人工转写的非指令片段,不使用其原任务标签作为 HFD 监督。AMI 另用人工对话行为标签排除请求、建议、提问等,按首末词边界保留完整语音。许可、原始标注、固定版本镜像及音频哈希均有记录;保守自动筛选不能替代人工听审。
|
| 18 |
+
|
| 19 |
+
## 保持与调整
|
| 20 |
+
|
| 21 |
+
| 项目 | 本版结果 |
|
| 22 |
+
|---|---|
|
| 23 |
+
| 数据总量 | train 26,800 / validation 784;82,590 turns;约 301.05 小时 |
|
| 24 |
+
| 最初构造方式(历史值) | 当时原音轨字节保留 13,792 条(50%),另 13,792 条更换背景;该比例不是后续音量修订后的字节保留率;speech_v3 完整成品保留 |
|
| 25 |
+
| 当前与 speech_v3 对比 | 音轨字节相同 12,440 条、不同 15,144 条;这是跨版比较,不是本次音量修订数量 |
|
| 26 |
+
| 本次音量修订前后对比 | 2,714 条仅提高背景,24,870 条字节保留;不改素材、位置、动作及上述配比;见[修复表](260927-third-party背景音量修复.md) |
|
| 27 |
+
| 目标与监督 | 构造时逐条保护原目标指令、动作时间线、总长度;保留原 source 室内底噪 |
|
| 28 |
+
| 三类人声 | 单人短插话 11,034;短三人声 5,517;持续 2~4 人声 11,033,约 40% / 20% / 40% |
|
| 29 |
+
| 位置 | 5,517 条全程覆盖;其余 22,067 条在各自合法起点区间内均匀随机,允许自然重叠指令 |
|
| 30 |
+
| 随机分布实测 | 合法起点区间等分十份,每份约占 9.82%~10.18%;不是强行按动作阶段分配比例 |
|
| 31 |
+
| 实际指令重叠 | 19,127 条,约 69.34%;由全程覆盖与随机放置共同产生,不是预设重叠配额 |
|
| 32 |
+
| 事件内片段复用 | 1,860 / 243,047 ≈ 0.77%;英文对话约 1.11%,禁止同一句紧邻循环 |
|
| 33 |
+
| 不新增 / 筛选目标 | 不新增 clean、额外白噪声/环境声;保守过滤动作指令、请求和提问/VQA 背景,人工复核待完成 |
|
| 34 |
+
|
| 35 |
+
## 验收与入口
|
| 36 |
+
|
| 37 |
+
| 项目 | 状态 / 入口 |
|
| 38 |
+
|---|---|
|
| 39 |
+
| 素材与试混 | 素材库校验、48 条真实短/长试混通过;215 项聚焦回归通过 |
|
| 40 |
+
| 数据隔离 | train/validation 的来源组、来源 speaker ID、原始/标准化音频哈希交叉均为 0 |
|
| 41 |
+
| 最初完整验收(历史) | 清单哈希、全量严格审计及一对一替换审计当时均通过;原报告见[修订前快照](data/hfd_third_party_cross_v4/production/reports/background_gain_before_reports.json) |
|
| 42 |
+
| 本次修订验收 | [独立全量验收](data/hfd_third_party_cross_v4/production/reports/background_gain_postcheck.json)、[完整严格审计](data/hfd_third_party_cross_v4/production/reports/strict_audit.json)通过,0 错误;替换报告与最终 GT 哈希已同步 |
|
| 43 |
+
| 交付入口 | 见 [hfd_third_party_current.json](hfd_third_party_current.json);不直接接入训练 |
|
| 44 |
+
| 新版成品 | [production](data/hfd_third_party_cross_v4/production) |
|
| 45 |
+
| 素材清单 | [registry](data/hfd_third_party_cross_v4/work/catalog/manifests/third_party_source_registry.jsonl) |
|
| 46 |
+
| 配额、使用量与复用率 | [cross_domain_summary.json](data/hfd_third_party_cross_v4/production/reports/cross_domain_summary.json) |
|
| 47 |
+
| 音频预览 | [24 组配对索引](data/hfd_third_party_cross_v4/production/manifests/preview_index.jsonl),覆盖两种语言 × 两类来源 × 三类干扰 × 两个 split |
|
| 48 |
+
| 人工听审 | [24 组配对试听](data/hfd_third_party_cross_v4/production/reports/listening_review.md);[475 条完整 session 清单](data/hfd_third_party_cross_v4/production/reports/expanded_listening_review.md),96 层、每层最多五条,少于五条则全选;均待审核 |
|
| 49 |
+
| GT 视频 | [12 类 × 2 条分类审核表](260927-third-party新版GT审核与清理.md),约 15 分钟;本次 03、05、09、13 已原位覆盖,其他 20 条保留;8 个更新视频全解码通过;10 的背景语义待复核 |
|
| 50 |
+
| 旧产物清理 | 经用户确认,仅在 HFD_lzk 内删除 112,259 个过时产物/重复副本,释放约 89.53 GiB;新版、speech_v3 成品、素材及 GT 保留,删除后核验通过;见[清理报告](260927-third-party新版GT审核与清理.md#旧产物清理) |
|
| 51 |
+
| 后续状态 | 人工试听待审核;training loader 未接入;`production_ready=false`,未训练 |
|
| 52 |
+
|
| 53 |
+
旧版 [人声重点版报告](260925-third-party人声重点版更新报告.md) 和其 GT 视频保留;那些视频仍属于旧版,不能当作本版新增素材的验收结果。
|
260927-third-party数据与GT速览.md
ADDED
|
@@ -0,0 +1,44 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# third-party 数据与 GT 速览
|
| 2 |
+
|
| 3 |
+
当前版本:`cross_v4_260926`。共 **27,584 条 session**(训练 26,800 / 验证 784),82,590 轮,约 301 小时。以下比例均按 session 计,取近似值。
|
| 4 |
+
|
| 5 |
+
## 1. 构造了什么
|
| 6 |
+
|
| 7 |
+
| 维度 | 当前构成 |
|
| 8 |
+
|---|---|
|
| 9 |
+
| 原始目标指令 | 中英文都有:中文 13,695(49.65%),英文 13,889(50.35%);这是本版选入的指令分布,不是背景比例 |
|
| 10 |
+
| 指令与动作 | 每条 2~4 轮,同一目标音色标签/语言(标签不保证实际声音一致);均为 turn-taking:听指令→执行对应动作→等待下一轮 |
|
| 11 |
+
| 背景语言与来源 | 中文对话 AliMeeting 30%;中文朗读 AISHELL-1 30%;英文对话 AMI 20%;英文朗读 LibriSpeech 20% |
|
| 12 |
+
| 干扰形态 | 单人短插话 11,034(40%);短三人声 5,517(20%);持续 2~4 人声 11,033(40%) |
|
| 13 |
+
| 背景位置 | 全程覆盖 5,517(20%);其余 22,067(80%)在可完整放入的起点范围内均匀随机,可能与指令重叠 |
|
| 14 |
+
| 素材与保留项 | 背景库为真人录音,26,027 段 / 1,069 个来源 speaker ID;保留原目标指令、动作监督及 source 自带室内底噪 |
|
| 15 |
+
| 筛选原则 | 背景以非指令人声为目标,过滤动作指令、请求、提问/VQA;不新增 clean 对照或额外白噪声,仍需人工语义审核 |
|
| 16 |
+
| 当前状态 | 响度修订的独立全量验收、完整严格审计和 GT 后验均通过;人工审核未完成、未接入训练 |
|
| 17 |
+
| 背景响度修订 | 原位提高 2,714 条;其余 24,870 条 recipe/WAV 字节不变,全部动作时间线及 44,141 项输入哈希不变;见[音量修复表](260927-third-party背景音量修复.md) |
|
| 18 |
+
|
| 19 |
+
## 2. “停止向前跑步”的歧义
|
| 20 |
+
|
| 21 |
+
| 要点 | 核对结果 |
|
| 22 |
+
|---|---|
|
| 23 |
+
| 出现位置 | **03** 的第二条目标指令;来自原始 TTS 文本 “stop a forward run / 停止向前跑步”,不是新增背景声 |
|
| 24 |
+
| GT 实际行为 | 鼓掌已结束并进入等待,听完该句后执行原始 `run_stop` 动作片段;没有紧急停止控制事件,并非“打断旧动作,再去跑步” |
|
| 25 |
+
| 问题判断 | 自然语言是在要求停止跑步,不能自然理解成开始跑步;把“跑步停止类动作描述”作为独立执行指令,会与真正的停止意图混淆,需在原始指令—动作配对层面复核 |
|
| 26 |
+
| 处理边界 | 真正的 Emergency Stop 应中断当前动作并进入 idle;该语义问题暂不处理,音量修订仅改背景,不改目标文本/音频和动作标签 |
|
| 27 |
+
|
| 28 |
+
原素材的英文也含 stop,所以不能只归因于中文翻译。**02、16、19** 也含停止类目标描述,建议一并复核;**10** 的背景有“多采购一点儿奖品”这类建议语气,仍待人工判定。
|
| 29 |
+
|
| 30 |
+
## 3. 01–24 对应类型
|
| 31 |
+
|
| 32 |
+
每格两例,均为条件随机抽样;视频只列编号。**前两行:前一个与指令重叠,后一个不重叠;最后一行:前一个全程覆盖,后一个局部覆盖。**
|
| 33 |
+
|
| 34 |
+
本次对应 **03、05、09、13** 的 session 已提高背景音量,GT 已原位覆盖;带/不带字幕共 8 个视频全解码通过,其余 20 条保留。
|
| 35 |
+
|
| 36 |
+
| 干扰类型 | 中文对话 | 中文朗读 | 英文对话 | 英文朗读 |
|
| 37 |
+
|---|---|---|---|---|
|
| 38 |
+
| 单人短插话(40%) | 01 / 02 | 03 / 04 | 05 / 06 | 07 / 08 |
|
| 39 |
+
| 短三人声(20%) | 09 / 10 | 11 / 12 | 13 / 14 | 15 / 16 |
|
| 40 |
+
| 持续 2~4 人声(40%) | 17 / 18 | 19 / 20 | 21 / 22 | 23 / 24 |
|
| 41 |
+
|
| 42 |
+
表头语言指**背景**,不是目标指令。审核视频按类别均衡抽样,不代表正式数据的中文 60% / 英文 40% 配比;GT 展示数据标签,不是模型效果。
|
| 43 |
+
|
| 44 |
+
核对依据:[正式构造清单](data/hfd_third_party_cross_v4/production/manifests/session_recipes.jsonl)、[GT 审核清单](data/hfd_third_party_cross_v4/gt_review_v2/review_manifest.json)及原始 TTS 元数据。
|
260927-third-party新版GT审核与清理.md
ADDED
|
@@ -0,0 +1,68 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# 新版 third-party GT 人工审核
|
| 2 |
+
|
| 3 |
+
版本:`cross_v4_260926`。本页记录 **24 条 GT 的首次交付及旧产物清理历史**,人工听审待完成;仅展示标签,不是模型推理,未训练。
|
| 4 |
+
|
| 5 |
+
本次背景响度修订已通过独立全量验收、完整严格审计及 GT 后验;**03、05、09、13 已原位覆盖**,带/不带字幕共 8 个更新视频全解码通过,其余 20 条原样保留,最终哈希已同步。最新状态见[音量修复表](260927-third-party背景音量修复.md)。
|
| 6 |
+
|
| 7 |
+
## 视频入口
|
| 8 |
+
|
| 9 |
+
每格 2 条,共 12 类 / 24 条,合计 14 分 59.6 秒。前两行每格前者与指令重叠、后者不重叠;最后一行前者全程覆盖、后者局部覆盖。点击编号观看。
|
| 10 |
+
|
| 11 |
+
| 干扰类型 | 中文对话 | 中文朗读 | 英文对话 | 英文朗读 |
|
| 12 |
+
|---|---|---|---|---|
|
| 13 |
+
| 单人短插话 | [01](data/hfd_third_party_cross_v4/gt_review_v2/01_foreground_non_command_zh_conversation_overlap/gt_with_audio_annotated.mp4)<br>[02](data/hfd_third_party_cross_v4/gt_review_v2/02_foreground_non_command_zh_conversation_nonoverlap/gt_with_audio_annotated.mp4) | [03](data/hfd_third_party_cross_v4/gt_review_v2/03_foreground_non_command_zh_read_overlap/gt_with_audio_annotated.mp4)<br>[04](data/hfd_third_party_cross_v4/gt_review_v2/04_foreground_non_command_zh_read_nonoverlap/gt_with_audio_annotated.mp4) | [05](data/hfd_third_party_cross_v4/gt_review_v2/05_foreground_non_command_en_conversation_overlap/gt_with_audio_annotated.mp4)<br>[06](data/hfd_third_party_cross_v4/gt_review_v2/06_foreground_non_command_en_conversation_nonoverlap/gt_with_audio_annotated.mp4) | [07](data/hfd_third_party_cross_v4/gt_review_v2/07_foreground_non_command_en_read_overlap/gt_with_audio_annotated.mp4)<br>[08](data/hfd_third_party_cross_v4/gt_review_v2/08_foreground_non_command_en_read_nonoverlap/gt_with_audio_annotated.mp4) |
|
| 14 |
+
| 短三人声 | [09](data/hfd_third_party_cross_v4/gt_review_v2/09_babble_zh_conversation_overlap/gt_with_audio_annotated.mp4)<br>[10](data/hfd_third_party_cross_v4/gt_review_v2/10_babble_zh_conversation_nonoverlap/gt_with_audio_annotated.mp4) | [11](data/hfd_third_party_cross_v4/gt_review_v2/11_babble_zh_read_overlap/gt_with_audio_annotated.mp4)<br>[12](data/hfd_third_party_cross_v4/gt_review_v2/12_babble_zh_read_nonoverlap/gt_with_audio_annotated.mp4) | [13](data/hfd_third_party_cross_v4/gt_review_v2/13_babble_en_conversation_overlap/gt_with_audio_annotated.mp4)<br>[14](data/hfd_third_party_cross_v4/gt_review_v2/14_babble_en_conversation_nonoverlap/gt_with_audio_annotated.mp4) | [15](data/hfd_third_party_cross_v4/gt_review_v2/15_babble_en_read_overlap/gt_with_audio_annotated.mp4)<br>[16](data/hfd_third_party_cross_v4/gt_review_v2/16_babble_en_read_nonoverlap/gt_with_audio_annotated.mp4) |
|
| 15 |
+
| 持续 2~4 人声 | [17](data/hfd_third_party_cross_v4/gt_review_v2/17_babble_continuous_zh_conversation_full/gt_with_audio_annotated.mp4)<br>[18](data/hfd_third_party_cross_v4/gt_review_v2/18_babble_continuous_zh_conversation_partial/gt_with_audio_annotated.mp4) | [19](data/hfd_third_party_cross_v4/gt_review_v2/19_babble_continuous_zh_read_full/gt_with_audio_annotated.mp4)<br>[20](data/hfd_third_party_cross_v4/gt_review_v2/20_babble_continuous_zh_read_partial/gt_with_audio_annotated.mp4) | [21](data/hfd_third_party_cross_v4/gt_review_v2/21_babble_continuous_en_conversation_full/gt_with_audio_annotated.mp4)<br>[22](data/hfd_third_party_cross_v4/gt_review_v2/22_babble_continuous_en_conversation_partial/gt_with_audio_annotated.mp4) | [23](data/hfd_third_party_cross_v4/gt_review_v2/23_babble_continuous_en_read_full/gt_with_audio_annotated.mp4)<br>[24](data/hfd_third_party_cross_v4/gt_review_v2/24_babble_continuous_en_read_partial/gt_with_audio_annotated.mp4) |
|
| 16 |
+
|
| 17 |
+
固定种子 `26092601`,每类 train / validation 各一例,在上述条件内随机抽取,不按时长或难度挑选。审核集为方便逐类比较而均衡抽样;正式数据仍为中文约 60%、英文约 40%。
|
| 18 |
+
|
| 19 |
+
## 首次自动验收(历史快照)
|
| 20 |
+
|
| 21 |
+
以下为音量修订前的验收结果,不代表本次覆盖后的最终验收。
|
| 22 |
+
|
| 23 |
+
| 项目 | 结果 |
|
| 24 |
+
|---|---|
|
| 25 |
+
| 视频 | 24 条;带字幕版 1280×720 / 50 FPS;总计 44,980 帧 |
|
| 26 |
+
| 音画与 GT | 24 条确定性时间线重建核对;带/不带字幕共 48 个含声视频完整解码通过 |
|
| 27 |
+
| 哈希 | 240 项产物/导出、48 项源音轨/时间线、原 10 项正式数据冻结哈希全部通过 |
|
| 28 |
+
| 可读性抽检 | 6 张画面;中英文三/四轨字幕无相互遮挡,2.5 米镜头改善全身取景 |
|
| 29 |
+
| 执行范围 | CPU 渲染,231 项聚焦回归通过;不改正式数据,不训练 |
|
| 30 |
+
| 详细记录 | [验收记录](data/hfd_third_party_cross_v4/gt_review_v2/handoff_audit.json);[抽帧目录](data/hfd_third_party_cross_v4/gt_review_v2/qa_frames);整套材料约 294 MiB |
|
| 31 |
+
|
| 32 |
+
## 怎么审
|
| 33 |
+
|
| 34 |
+
| 画面 / 检查项 | 含义 / 要看什么 |
|
| 35 |
+
|---|---|
|
| 36 |
+
| 绿色目标指令 | 动作是否与目标对应,时间是否衔接合理 |
|
| 37 |
+
| 橙色第三方人声 | 是否为非指令、非 VQA;中英文、音色与多人声是否自然;目标是否可辨 |
|
| 38 |
+
| 黄色控制阶段 | listen / motion / idle 是 GT 标签,不能据此证明模型抗干扰效果 |
|
| 39 |
+
| 多人字幕 | 持续人声逐轨随片段更新;短三人声为事件级字幕,不代表三句同时开口;长句会截略 |
|
| 40 |
+
| 问题记录 | 请给出视频编号、时间点、问题描述;完整文本及时间见各样例 session_recipe.json |
|
| 41 |
+
|
| 42 |
+
自动验收状态、详细来源和每例文件说明见 [GT 清单](data/hfd_third_party_cross_v4/gt_review_v2/README.md);人工结论仍为待审核。
|
| 43 |
+
|
| 44 |
+
### 优先复核
|
| 45 |
+
|
| 46 |
+
| 样例 / 区间 | 观察 | 待人工确认 |
|
| 47 |
+
|---|---|---|
|
| 48 |
+
| 10 / 约 14.33~17.65 秒 | 背景含“多采购一点儿奖品”,有建议/祈使语气 | 虽非机器人动作指令,是否仍应按严格“非指令”要求剔除;此处只标注,不修改数据或换掉随机样例 |
|
| 49 |
+
|
| 50 |
+
自动语义筛选不能替代人工复核;自动校验通过不等于人工语义审核通过。具体素材 ID 见[问题记录](data/hfd_third_party_cross_v4/gt_review_v2/review_findings.json)。
|
| 51 |
+
|
| 52 |
+
## 旧产物清理
|
| 53 |
+
|
| 54 |
+
以下为已完成清理的历史记录;其中“不变”及哈希核验均指清理前后,不用于证明后续音量修订不变。全部删除目标位于 LZK 独立工作目录 `code/HFD_lzk` 内,逐文件排除越界及软硬链接;未触及师兄的 `code/HFD` 或共享输入。
|
| 55 |
+
|
| 56 |
+
| 已清理范围 | 文件数 | 释放(GiB,按已删除文件的分配块统计) |
|
| 57 |
+
|---|---:|---:|
|
| 58 |
+
| scale10pct_v1 旧成品 audio / timelines / previews | 55,204 | 39.23 |
|
| 59 |
+
| diverse_v2 旧成品 audio / timelines / previews | 55,204 | 39.29 |
|
| 60 |
+
| 旧 pilot 及新版试混 smoke 的 audio / timelines / previews | 1,812 | 1.33 |
|
| 61 |
+
| AISHELL 已合并下载分片、AMI 临时重复镜像分片 | 39 | 9.68 |
|
| 62 |
+
| 合计 | **112,259** | **89.53(约 96.13 GB)** |
|
| 63 |
+
|
| 64 |
+
保留 cross_v4 和 speech_v3 完整成品、全部原始/标准化素材及来源文件(仅清除上述重复下载副本)、历史构造清单/报告和所有 GT 视频。158,526 个保护文件前后属性一致;13 项当前冻结哈希、24 条 GT 的 312 项文件哈希复查通过;28 项聚焦测试通过。9 个旧产物目录已添加退役标记,旧审计报告仅作历史记录。
|
| 65 |
+
|
| 66 |
+
本次为永久删除,无回收站副本;旧成品可依据保留的构造记录和原始素材重新生成。释放量不是共享文件系统全局剩余空间的差值。
|
| 67 |
+
|
| 68 |
+
详见[执行结果](data/hfd_third_party_cross_v4/work/cleanup_260926_result.json)、[删除后验收](data/hfd_third_party_cross_v4/work/cleanup_260927_postcheck.json)及[逐文件计划](data/hfd_third_party_cross_v4/work/cleanup_260926_plan.json)。GT 验收记录的 `pre_audio_revision_audit` 保留首次交付快照,清理最终状态以本节及执行结果为准;本次音量修订另见[GT 后验](data/hfd_third_party_cross_v4/work/background_gain_gt_postcheck.json)。
|
260927-third-party背景音量修复.md
ADDED
|
@@ -0,0 +1,41 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# third-party 背景音量修复
|
| 2 |
+
|
| 3 |
+
版本:`cross_v4_260926` 原位音量修订。只提高偏轻的第三方人声,不改目标指令、动作、素材、位置、时长和中英配比;不新建数据集目录,不训练。
|
| 4 |
+
|
| 5 |
+
| 项目 | 结果 |
|
| 6 |
+
|---|---:|
|
| 7 |
+
| 全量检查 | 27,584 条,修复前逐采样及 WAV 哈希全部吻合 |
|
| 8 |
+
| 背景音量提高 | 2,714 条(9.84%) |
|
| 9 |
+
| 完全不改 | 24,870 条 |
|
| 10 |
+
| 调整类型 | 单人 1,838;短三人 476;持续多人 400 |
|
| 11 |
+
| 现有 GT | 03、05、09、13 已原位覆盖;其他 20 条视频原样保留 |
|
| 12 |
+
| 配对音频预览 | 同步更新 4 组的增强音频,base 不改 |
|
| 13 |
+
| 独立全量验收 | 通过:27,584 条逐条核验,0 失败 |
|
| 14 |
+
| 不变性核验 | 24,870 条 recipe/WAV 字节不变;27,584 条动作时间线、44,141 项输入文件哈希不变 |
|
| 15 |
+
| 完整严格审计 | 通过,0 错误;一对一替换报告与最终 GT 哈希已同步 |
|
| 16 |
+
| 回归与 GT 后验 | 163 项测试通过;带/不带字幕共 8 个更新视频全解码及独立 GT 后验通过 |
|
| 17 |
+
|
| 18 |
+
## 怎么调整
|
| 19 |
+
|
| 20 |
+
| 检查 | 处理 |
|
| 21 |
+
|---|---|
|
| 22 |
+
| 背景比“完整重叠指令”参照下的原目标强度低超过 3 dB | 用完整指令定标,不再跟随弱尾音;无重叠时参考全 session 目标指令 |
|
| 23 |
+
| 背景整体 RMS 不足原底噪 +3 dB | 尝试提高至底噪 +6 dB,同时限制对目标指令的掩盖 |
|
| 24 |
+
| 安全边界 | 只向原 PCM 加背景增量,不全段归一化;峰值不超过约 0.95,不削波 |
|
| 25 |
+
|
| 26 |
+
3 条样本受峰值限制,已安全提高但未达到预设困难档强度;其背景已高于底噪约 15.8~26.6 dB,保留人工复核标记。自动响度判定不等于人工听审通过。
|
| 27 |
+
|
| 28 |
+
## 需要重听的 GT
|
| 29 |
+
|
| 30 |
+
以下是已修复 session 中,同一背景事件区间内“第三方人声整体 RMS 相对原底噪”的分贝差;正数表示人声更强。对应 GT 已复用原动作画面,原位更新声音与相关字幕。
|
| 31 |
+
|
| 32 |
+
| 编号 | 修复前 | 修复后 |
|
| 33 |
+
|---|---:|---:|
|
| 34 |
+
| 03 | −70.7 dB | +6.0 dB |
|
| 35 |
+
| 05 | −62.5 dB | +5.7 dB |
|
| 36 |
+
| 09 | −3.2 dB | +7.9 dB |
|
| 37 |
+
| 13 | −4.3 dB | +17.6 dB |
|
| 38 |
+
|
| 39 |
+
09 的三路人声仍在 **26.27~28.52 秒**,只改变响度。03 的停止指令语义、06 的目标音色一致性等问题,本轮没有处理。
|
| 40 |
+
|
| 41 |
+
审计入口:[独立全量验收](data/hfd_third_party_cross_v4/production/reports/background_gain_postcheck.json)、[严格审计](data/hfd_third_party_cross_v4/production/reports/strict_audit.json)、[GT 后验](data/hfd_third_party_cross_v4/work/background_gain_gt_postcheck.json)、[逐条修复记录](data/hfd_third_party_cross_v4/production/reports/background_gain_revision.json)。保留压缩的原构造清单以便精确回放;session ID 沿用原身份,最新音频以修订后的 SHA256 为准。
|
260928-third-party多段人声更新报告.md
ADDED
|
@@ -0,0 +1,83 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# third-party 统一训练交付说明
|
| 2 |
+
|
| 3 |
+
更新:2026-10-08。用于给机器人补充“听从真实指令、忽略旁人说话”的能力。300条过拟合实验已完成;后续交给主项目统一混训,LZK的独立全量训练和性能短测均已停止。
|
| 4 |
+
|
| 5 |
+
## 交付路径
|
| 6 |
+
|
| 7 |
+
项目根目录(下表路径均相对此处):
|
| 8 |
+
|
| 9 |
+
```text
|
| 10 |
+
/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/HFD_lzk
|
| 11 |
+
```
|
| 12 |
+
|
| 13 |
+
| 用途 | 路径 |
|
| 14 |
+
| ---------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------- |
|
| 15 |
+
| **过拟合预测视频** | [runs/third_party_original16_20261008_0038/prediction_videos/](runs/third_party_original16_20261008_0038/prediction_videos/README.md),查看 `01.mp4`~`24.mp4` |
|
| 16 |
+
| **筛选后的数据全集** | [data/hfd_third_party_multi_v5/production/](data/hfd_third_party_multi_v5/production),27,520条 |
|
| 17 |
+
| **建议并入的训练清单** | `selection.json`:26,737条(`data/hfd_third_party_multi_v5/training_scale_all_commands/selection.json`) |
|
| 18 |
+
| **独立验证清单** | `validation.json`:783条(`data/hfd_third_party_multi_v5/training_scale_all_commands/validation.json`) |
|
| 19 |
+
| 过拟合指标和loss曲线 | [runs/third_party_original16_20261008_0038/review.md](runs/third_party_original16_20261008_0038/review.md) |
|
| 20 |
+
| 数据GT视频,供核对标签 | [data/hfd_third_party_multi_v5/gt_review_v1/README.md](data/hfd_third_party_multi_v5/gt_review_v1/README.md) |
|
| 21 |
+
|
| 22 |
+
## 数据量与类型
|
| 23 |
+
|
| 24 |
+
`production` 已清理为 **27,520条、300.36小时**:**训练26,737条/291.87小时,验证783条/8.49小时**,与交付清单一致。原构造27,584条、301.05小时,其中64条质量边界样本已删除音频和标签文件。
|
| 25 |
+
|
| 26 |
+
| 类型与代码名 | 原总量/约占比 | 交付训练 | 交付验证 | 预测视频编号 |
|
| 27 |
+
| ---------------------------------- | -------------: | -------: | -------: | ------------ |
|
| 28 |
+
| 单段短人声`single_short` | 4,138/15% | 3,997 | 117 | 01–04 |
|
| 29 |
+
| 多段单人声`multi_single` | 6,896/25% | 6,700 | 196 | 05–08 |
|
| 30 |
+
| 多段多人声`multi_babble` | 4,138/15% | 4,020 | 118 | 09–12 |
|
| 31 |
+
| 单人和多人混合`mixed_short` | 8,550/31% | 8,308 | 242 | 13–16 |
|
| 32 |
+
| 连续多人背景`continuous` | 2,758/10% | 2,667 | 78 | 17–20 |
|
| 33 |
+
| 指令同音色旁话`same_voice_aside` | 1,104/4% | 1,045 | 32 | 21–24 |
|
| 34 |
+
| 合计 | 27,584/100% | 26,737 | 783 | 24条 |
|
| 35 |
+
|
| 36 |
+
每组4条预测视频依次为“中文普通音量、中文较高音量、英文普通音量、英文较高音量”。
|
| 37 |
+
|
| 38 |
+
背景语言原本中英各50%;交付训练为中文13,368/英文13,369,验证392/391。目标指令本身也有中英文,二者不要混淆。
|
| 39 |
+
|
| 40 |
+
## 怎么构造
|
| 41 |
+
|
| 42 |
+
保留原source的目标指令、动作配对及时间线,在原音轨上叠加第三方人声;模型应按真实指令动作,不应因背景说话误动。
|
| 43 |
+
|
| 44 |
+
| 项目 | 实际做法 |
|
| 45 |
+
| ------------ | -------------------------------------------------------------------------------------------------------------------------------------------- |
|
| 46 |
+
| 真人素材 | 中文:AliMeeting会议对话+AISHELL-1朗读;英文:AMI会议对话+LibriSpeech朗读。素材池中英各4,494段,原构造用到8,223段、1,065个来源说话人ID |
|
| 47 |
+
| 同音色旁话 | 用session中一条实际指令音频作TTS音色参考,合成非指令旁话。共75段(中41/英34),复用于1,104条原session,并搭配真人背景;不是仅按音色标签匹配 |
|
| 48 |
+
| 每条背景构成 | 单段1段;多段单人2–4段;多段多人2–3段;混合3–5段;连续多人1段连续声场;同音色旁话1段+真人背景1–2段 |
|
| 49 |
+
| 插入位置 | 普通真人背景在整个session内随机起始,可与指令或其他背景重叠,允许到结尾截断。同音色旁话在参考指令之后、不与真实指令重叠的区间随机插入 |
|
| 50 |
+
| 音量 | “指令减整体背景”的有效声音差按截断正态`N(2,3²) dB,范围[-4,8]`抽取;实测背景平均低2.00dB,约24.16%比指令��响。限峰时整段同比缩放 |
|
| 51 |
+
| 内容边界 | 对背景转写过滤动作指令、请求及提问/VQA;不另设clean类别、不新增白噪声,保留source已有底噪。多人声可由片段叠加,不保证对话连贯 |
|
| 52 |
+
| 可复现 | 原session组合seed为26091703,背景seed为26092801;每条的具体素材、位置、音量及时间线seed均写入recipe |
|
| 53 |
+
|
| 54 |
+
## 对应代码与并训接入
|
| 55 |
+
|
| 56 |
+
数据构造入口是 [build_hfd_third_party_multi_v5.py](scripts/data/build_hfd_third_party_multi_v5.py);并训直接使用现成数据,无需重新构造。
|
| 57 |
+
|
| 58 |
+
1. **接数据:**复用 [ThirdPartyFrozenDataset](src/psi/data/seed/third_party_frozen_dataset.py),将 `production_root` 指向上面的 `production`,`selection_path` 指向训练清单 `selection.json`,使用 `split="train"`。读取器返回已混好的人声输入和原动作/控制等监督,不再随机组合或叠加背景;保留其依赖的原source路径。验证783条留作单独评估。
|
| 59 |
+
2. **接训练:**参考 [ThirdPartyDataConfig](src/psi/config/train/posttrain_third_party_config.py) 的数据包装方式,把third-party作为一个数据源加入师兄的混合采样。训练逻辑沿用 [PosttrainFullDuplexTrainer](src/psi/trainers/posttrain_full_duplex.py) 和师兄统一的训练配置,不照搬本次300条专用启动脚本及限制。
|
| 60 |
+
|
| 61 |
+
## 89万步过拟合实验参考
|
| 62 |
+
|
| 63 |
+
| 项目 | 方法 |
|
| 64 |
+
| ----------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
|
| 65 |
+
| 数据 | 六类各50条,共300条;只用third-party语音,不混其他任务或视觉,最终也在这300条上评估 |
|
| 66 |
+
| 初始化 | 联合阶段承接此前793374模型及校正后的控制头,重建优化器/调度;不是从随机权重直接训练89万步 |
|
| 67 |
+
| 训练范围 | 沿用v24损失,联合更新Qwen末两层和norm、动作网络、控制头及时长等模块 |
|
| 68 |
+
| batch和覆盖 | batch=16;相近长度分组,每轮打乱组顺序,每条完整覆盖一次,保留尾批。联合阶段20遍,每遍44,531次更新,合计890,620步 |
|
| 69 |
+
| 学习率 | 动作2e-5、控制5e-6、Qwen5e-7;预热500步,余弦衰减至峰值10% |
|
| 70 |
+
| 监督记录 | 每轮固定训练60条看指标,结束评估全部300条,随机渲染24条预测视频;[训练代码](scripts/train_third_party_convergence.py)、[续训代码](scripts/resume_third_party_convergence.py) |
|
| 71 |
+
|
| 72 |
+
| 最终同300条上的指标 | 结果 |
|
| 73 |
+
| ---------------------------- | ---------------------------------: |
|
| 74 |
+
| 控制阶段准确率 | 96.40% |
|
| 75 |
+
| 背景非动作时段误动率 | 14.32%(逐条平均);逐帧加权14.87% |
|
| 76 |
+
| 应动作时被预测为非动作的比例 | 1.07% |
|
| 77 |
+
| 关节角平均绝对误差 | 0.17355 rad |
|
| 78 |
+
|
| 79 |
+
背景误动率只在“背景有声、GT不要求动作”的区间计算;300条中265条有可评分区间。[完整结果](runs/third_party_original16_20261008_0038/final_train300/report.json)。
|
| 80 |
+
|
| 81 |
+
**与全量混训的区别:**本次先在已有模型上优化控制头,再联合训练89万多步;优化只改变控制头参数,没有改变结构或数据标签。并入师兄训练后,可按原初始化方案重新混训(保留预训练Qwen底座,不继承本次控制头或89万步权重)。third-party提供的是“背景有人说话,但仍按真实指令动作”的监督,与原任务目标一致,因此从头混训在方法上合理;现有过拟合效果支持这部分数据有学习价值,但不能保证新初始化下直接复现相同效果。
|
| 82 |
+
|
| 83 |
+
结论:已显示一定抗第三方干扰能力,可作为并入统一训练的依据;动作质量仍有不足,上述结果不是任务成功率或测试集泛化成绩,也未覆盖此次恢复的停止类样本。
|
260930-third-party训练实验计划.md
ADDED
|
@@ -0,0 +1,68 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# third-party 训练实验计划(待审核,未开训)
|
| 2 |
+
|
| 3 |
+
> 2026-10-02 更新:不再要求师兄的 HFD checkpoint,改为新初始化 HFD 任务模块;暂按保留 Qwen 预训练底座理解,待确认。300 条阶段用 1 张 140GB H200,全量用 1 或 2 张。下文关于加载已训 HFD 基线的旧方案不再执行,操作以[新版手册](261002-third-party训练实验操作手册.md)为准。
|
| 4 |
+
|
| 5 |
+
目标:**只用 third-party 特训:300条同训集看能否学会 → 全量训练 → 保留集看抗干扰效果。** 暂不混师兄其他任务数据;沿用原动作/TTS底层素材不等于混训。产物均在 HFD_lzk。
|
| 6 |
+
|
| 7 |
+
## 1. 沿用什么,先补什么
|
| 8 |
+
|
| 9 |
+
已核对正式入口:[posttrain-full-duplex-audio-command.sh](posttrain-full-duplex-audio-command.sh)。沿用其 **v24模型、trainer、损失及KV/RTC时序**,增加薄封装和必要接口,方便以后统一混训;不用7task脚本,也不照搬默认v12/50-source的旧overfit脚本。
|
| 10 |
+
|
| 11 |
+
| 开训前必须补齐 | 当前情况/方案 |
|
| 12 |
+
|---|---|
|
| 13 |
+
| 冻结数据入口 | loader尚未接入v5。按固定session清单读最终混音WAV,不在线重组或重复加背景;不能仅改目录/限制条数 |
|
| 14 |
+
| 完整监督 | NPZ只是精简审计时间线。按recipe恢复完整动作尾段及控制/时长/文本监督,并校验音频哈希、时间轴;保留隔离检查,不靠降版本绕过合同 |
|
| 15 |
+
| 基线模型 | 师兄指定兼容v24的HFD checkpoint;原脚本不会自动加载已训练的动作/控制模型。审核兼容后作权重初始化,新建run/优化器,不误续旧训练状态 |
|
| 16 |
+
| 冻结音频推理 | 原推理会重混/重拼音频,需补“整段WAV原样流式输入”,复用既有模型推理和视频渲染 |
|
| 17 |
+
|
| 18 |
+
## 2. 300 条怎么选
|
| 19 |
+
|
| 20 |
+
一条轨迹=一个完整多轮 **session**,不是一帧。只从现有 **train** 中选:
|
| 21 |
+
|
| 22 |
+
| 类型 | 小集 | 全量train |
|
| 23 |
+
|---|---:|---:|
|
| 24 |
+
| 单段单人声 | 50 | 4,020 |
|
| 25 |
+
| 多段单人声 | 50 | 6,700 |
|
| 26 |
+
| 多段多人声 | 50 | 4,020 |
|
| 27 |
+
| 单人+多人混合 | 50 | 8,308 |
|
| 28 |
+
| 连续多人声 | 50 | 2,680 |
|
| 29 |
+
| 同音色旁话+真人背景 | 50 | 1,072 |
|
| 30 |
+
| **合计** | **300** | **26,800** |
|
| 31 |
+
|
| 32 |
+
每类背景中25/英25,高背景约12/低背景38(贴近生产约24%更响,不照搬GT审核的各半);同音色尽量覆盖不同参考素材。固定抽样seed、ID和哈希,不改原构造seed、声音或位置,只写引用清单。小集避开技术无效背景及经人工确认的指令歧义;候选充足。**GT24不是训练全集,编号23属validation,不能混入。**
|
| 33 |
+
|
| 34 |
+
## 3. 实验顺序与预算
|
| 35 |
+
|
| 36 |
+
| 阶段 | 预算建议 | 检查/输出 |
|
| 37 |
+
|---|---|---|
|
| 38 |
+
| 接入验收 | CPU校验300条 | 实际batch仅来自清单,音频与完整监督对齐 |
|
| 39 |
+
| GPU smoke | 20–50 optimizer steps;每卡先2–4个session | 梯度/KV/保存恢复/预测链路正常,实测显存和吞吐后定正式batch及卡时 |
|
| 40 |
+
| 小集过拟合 | 初始 **3S steps**,必要时经确认延至5S | **同一300条训练集**看loss、控制与预测动作是否改善;未学会先排查,不直接扩量 |
|
| 41 |
+
| 全量特训 | 从同一原始基线重新开始;先完整覆盖1遍,再评审是否扩至3遍 | 用26,800条train、恢复生产配比,不沿用300条过拟合权重 |
|
| 42 |
+
| 最终评估 | 冻结现有 **784条validation**,训练配置/预算/选checkpoint规则先固定 | 同输入和推理seed比较基线/特训后;按六类、语言、音量汇总 |
|
| 43 |
+
|
| 44 |
+
**S=完整覆盖300条的实际optimizer步数**,按最终清单、获准GPU、实际全局batch及采样器核定;约58万密集起点只是估算,不是300步。覆盖以每条session完整走完计,不只是见过ID。warmup拟总步数1%,不照搬125万步/12,500步warmup;约每S/2在安全source边界保存checkpoint。
|
| 45 |
+
|
| 46 |
+
同训集评估单列 `train_probe`,不冒充validation、不关闭防泄漏保护,也不把过拟合当泛化。全量前须确认37条整体背景不可测、27条短克隆及歧义指令的处理;不静默删改,若批准排除则另报实际训练数。
|
| 47 |
+
|
| 48 |
+
**目前没有独立test。** 本轮784条封存至最终评估;训练时关闭默认validation、best_ckpt选优和自动val套件,只做train_probe。若反复用784条选型,须另冻结最终test。当前动作实例/背景来源隔离,但目标指令WAV跨split复用,结果不能宣称未见指令声音的全面泛化。
|
| 49 |
+
|
| 50 |
+
## 4. 训练中要能看到什么
|
| 51 |
+
|
| 52 |
+
| 展示 | 计划(新增部分尚待实现) |
|
| 53 |
+
|---|---|
|
| 54 |
+
| 曲线 | 复用WandB标量:总loss、action/control/duration/文本分项、准确率、LR、梯度、显存/吞吐;补session覆盖数 |
|
| 55 |
+
| 视频面板 | 从300条固定12条(每类2,兼顾中英/音量)。训练前及checkpoint点展示 **预测+GT+原混音音轨+控制时间线**,固定输入及推理seed |
|
| 56 |
+
| 同训集检查 | 小集结束评估全部300条;常规只渲染固定12条及失败例,节约时间和空间 |
|
| 57 |
+
| 抗干扰指标 | 六类分报背景误触发/误中断、真实���令漏响应/延迟、动作误差/跳变及观感;按GT与目标语音区间计分,重叠情况单列 |
|
| 58 |
+
| 留档 | 本地保存曲线/表格/MP4及run、session、输入哈希、step。已有标量,视频面板/新指标待补;个人WandB音视频上传需确认,不联网则先补本地/离线分支 |
|
| 59 |
+
|
| 60 |
+
视频必须是**真实闭环预测**:仅允许初始姿态,后续control/state/RTC来自预测;GT只评分/并排显示,不能用GT渲染冒充效果。
|
| 61 |
+
|
| 62 |
+
## 5. 占卡与审核后再做的事
|
| 63 |
+
|
| 64 |
+
- 原launcher会停止/恢复**整个共享monitor**,不只影响本次卡号。拟加显式管理开关、默认不接管;先约定卡号/时段/权限,不照抄8卡示例,不停止他人任务。
|
| 65 |
+
- 训练/推理显式绑定获准卡及LZK run,不裸跑自动套件;无推理专卡就分阶段评估。退出后只按授权约定恢复占卡。
|
| 66 |
+
- 待你们确认:**基线checkpoint、300条与3S预算、784条保留评估方案、GPU/占卡权限、WandB共享范围**。确认后再补接口、抽样和smoke;本次只写规划,未改训练代码、未开训、未占卡。
|
| 67 |
+
|
| 68 |
+
依据:[9月4日交接](260904-lzk交接文档.md) · [数据与GT报告](260928-third-party多段人声更新报告.md) · [当前数据配置](src/psi/config/data_full_duplex_audio_command.py) · [原推理实现](scripts/infer/full_duplex_audio_to_motion.py)
|
261002-third-party训练实验操作手册.md
ADDED
|
@@ -0,0 +1,232 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# third-party 训练操作手册
|
| 2 |
+
|
| 3 |
+
## 当前安排:交付数据,停止独立训练(2026-10-08)
|
| 4 |
+
|
| 5 |
+
独立全量训练和性能短测均已停止,GPU0占卡已恢复并核验,既有权重和结果保留。后续由师兄把third-party并入统一训练;下面的启动/续训步骤仅作历史参考,不要重复开训。
|
| 6 |
+
|
| 7 |
+
交付训练26,737条、验证783条,保留真实“停止/stop”指令。数据路径、构造代码、类型分布和过拟合方法见[交付说明](260928-third-party多段人声更新报告.md);审核效果见[890620模型的24条预测视频](runs/third_party_original16_20261008_0038/prediction_videos/README.md)。
|
| 8 |
+
|
| 9 |
+
## 已完成的300条实验(以下为历史记录)
|
| 10 |
+
|
| 11 |
+
按你的要求,从切换 batch 之前的 `third_party_joint_convergence_20261007_0127/checkpoints/ckpt_465760` 完整恢复,继续原 batch=16。仍只用原 300 条 third-party,无其他任务、视频或保留集输入。起点是本阶段已完成 10 遍、第 11 遍已完成 160 条;不是从头训练,也不接着 batch=32 的权重训练。
|
| 12 |
+
|
| 13 |
+
**本轮已于 10 月 8 日 10:21 正常结束:890620 步、联合阶段累计 20 遍,最终训练集 300 条评估已完成,占卡已恢复并核验。** 停止原因是 `stop_budget`(达到预算),不是已经收敛。batch=32 分支的权重和指标未混入本轮;旧权重和结果保留,不要重复开训。
|
| 14 |
+
|
| 15 |
+
**看动作视频:**打开 [最终预测视频索引](runs/third_party_original16_20261008_0038/prediction_videos/README.md),或从原 `review.md` 底部进入。**24 条已全部完成并核验**,按六类×中英文背景×普通/较高背景音量随机抽取,共约 17 分钟。这里是最终 **890620 模型的预测,不是 GT**,也不是第 15 遍的最优候选权重。使用已有预测做 CPU 并行渲染,没有重新推理或停占卡。
|
| 16 |
+
|
| 17 |
+
| 设置 | 本次方案 |
|
| 18 |
+
|---|---|
|
| 19 |
+
| 当前状态 | 已完成 890620 步、20 遍;`stop_budget`,尚未满足收敛准则 |
|
| 20 |
+
| 恢复内容 | 从 `ckpt_465760` 恢复模型、优化器、学习率、随机状态及数据游标;不重新套用控制头、不重置优化器 |
|
| 21 |
+
| 联合训练 | Qwen 末两层和 norm、动作网络、控制头、时长头;不是只训控制头 |
|
| 22 |
+
| 并行策略 | 固定原 batch=16、数据加载 8 进程;评估仍 4 进程,不再按吞吐自动选 batch |
|
| 23 |
+
| 每遍覆盖 | 300 条各一次,六类各 50;第 11 遍只补剩余 140 条,保留尾批、不重复填充 |
|
| 24 |
+
| 预算与顺序 | 原分组和顺序不变,完整一遍 44531 次更新;最多再更新 424860 步至原上限 890620,不重新增加 20 遍 |
|
| 25 |
+
| 学习率 | 恢复原调度状态和原曲线,不重启预热、不压缩或重新映射衰减 |
|
| 26 |
+
| 收敛检查 | 每遍完整有效样本加权的动作 loss/控制 CE,连续三次改善均不足 1% 且不恶化,再检查闭环指标 |
|
| 27 |
+
| 效果复查 | 每遍固定训练 60 条+其中配对去旁话 24 条;中途不渲染视频 |
|
| 28 |
+
| 存档 | 每组完整轨迹后保存,本目录滚动保留三份全状态;保留旧实验与原最优模型,首个新存档实际重载验收 |
|
| 29 |
+
| 占卡 | 管理器自动交接,结束/报错后自动恢复,并记录是否核验成功 |
|
| 30 |
+
|
| 31 |
+
进入环境并指定**本次**目录:
|
| 32 |
+
|
| 33 |
+
```bash
|
| 34 |
+
cd /cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/HFD_lzk
|
| 35 |
+
source .venv-psi/bin/activate
|
| 36 |
+
export HFD_TRAIN_RUN="$PWD/runs/third_party_original16_20261008_0038"
|
| 37 |
+
python scripts/train/third_party_run.py status --run-dir "${HFD_TRAIN_RUN:?请先设置目录}"
|
| 38 |
+
python -m json.tool "$HFD_TRAIN_RUN/workflow_state.json"
|
| 39 |
+
```
|
| 40 |
+
|
| 41 |
+
`restoring_formal_training` 是载入原存档及准备数据;`joint_training` 是更新参数;`epoch_train60_metrics`/`epoch_paired24_metrics` 是轮末推理;`final_train300_metrics` 是最终全训练集复查。本次不再运行并行选型测试,步数从 465760 按原计划接续;batch=32 分支的步数、指标不混入本次曲线。
|
| 42 |
+
|
| 43 |
+
| 查看内容 | 文件 |
|
| 44 |
+
|---|---|
|
| 45 |
+
| 总入口/曲线/逐轮指标 | `review.md` |
|
| 46 |
+
| 最适合判断 loss 趋势的逐轮曲线 | `epoch_losses.svg`(继承此前 10 遍并继续记录) |
|
| 47 |
+
| 实时分项曲线/原始标量 | `curves.svg`/`metrics.jsonl` |
|
| 48 |
+
| 每遍统计/停止理由 | `epoch_metrics.jsonl`/`convergence.json` |
|
| 49 |
+
| 当前遍数、步数、已完成轨迹 | `workflow_state.json`/`coverage_state.json` |
|
| 50 |
+
| 单条及分组推理结果 | `metrics_evaluations/step_*/report.json`;对应 `clean/report.json` |
|
| 51 |
+
| 最终完整 300 条指标 | `final_train300/report.json`(原报告中 0 条渲染是推理结束时的记录,后来的视频另存下行) |
|
| 52 |
+
| 最终预测视频/类型表/渲染进度 | `prediction_videos/README.md`;视频 `01.mp4`–`24.mp4`;`prediction_videos/status.json` |
|
| 53 |
+
| GPU 利用率/日志 | `gpu_usage.jsonl`/`train.log` |
|
| 54 |
+
| 原配置/恢复核验 | `chosen_strategy.json`/`full_state_resume_verified.json`;本次没有性能测试文件 |
|
| 55 |
+
|
| 56 |
+
查看实时日志:`tail -n 50 -f "$HFD_TRAIN_RUN/train.log"`,Ctrl-C 只退出查看。
|
| 57 |
+
|
| 58 |
+
确需停止时执行:
|
| 59 |
+
|
| 60 |
+
```bash
|
| 61 |
+
python scripts/train/third_party_run.py stop --run-dir "${HFD_TRAIN_RUN:?请先设置目录}"
|
| 62 |
+
```
|
| 63 |
+
|
| 64 |
+
不要手动 `pkill`。退出后查 `keeper_restore_verified: true`;重启/断电不能保证自动恢复。若服务器重启,保留本次 checkpoint、`coverage_state.json`、`coverage_plan.json` 后让助手核对恢复,不能套用旧 batch=2 续训入口。
|
| 65 |
+
|
| 66 |
+
`completed` 只表示工作流结束,不等于收敛:`result.json` 中 `stop_plateau` 才表示满足本轮平台准则;`stop_budget` 是预算用完;`stop_regression` 是效果退化保护。平台也不等于动作质量过关,更不能代表测试集泛化。
|
| 67 |
+
|
| 68 |
+
---
|
| 69 |
+
|
| 70 |
+
## 以下为 10 月 6 日原三遍训练的历史操作记录(不用于当前任务)
|
| 71 |
+
|
| 72 |
+
更新:2026-10-06。**已按你的授权切换到并行策略流程:实测并行度 → 补齐历史 60 条指标 → 加大 batch 续训 → 最终 300 条指标。不要重复启动,也不要删除旧实验。** 始终只使用固定的 300 条 third-party,无测试集或视频输入。
|
| 73 |
+
|
| 74 |
+
| 当前情况 | 已核实 |
|
| 75 |
+
|---|---|
|
| 76 |
+
| 本次训练目录 | `runs/third_party_parallel_continue_20261006_1828` |
|
| 77 |
+
| 已完成的历史补评估 | `runs/third_party_parallel_20261006_174755/history`;三份现存保存点均 60/60,本次汇总页直接链接 |
|
| 78 |
+
| 恢复点 | 上轮 `third_party_resume_20261005_235100/checkpoints/ckpt_770084` |
|
| 79 |
+
| 恢复内容 | 模型、优化器、随机状态;学习率从存档值接续,并按新的剩余更新次数衰减 |
|
| 80 |
+
| 数据预算 | 只完成第 3 遍剩余的 148 条轨迹,不额外增加训练遍数 |
|
| 81 |
+
| 已选策略 | 训练 batch=16,推理 4 进程;测试更新不作为正式权重 |
|
| 82 |
+
| 当前进展 | 历史三份 60 条补评估已完成;完整恢复与首批 16 条轨迹通过,正在后台正式续训 |
|
| 83 |
+
| 新训练终点 | 793374 次累计优化更新:从 770084 再更新 23290 次,完成剩余 148 条 |
|
| 84 |
+
| 中途评估 | 固定 60 条,每类 10 条、中英各 5 条;只存指标和预测轨迹,不渲染 |
|
| 85 |
+
| 最终评估 | 全部 300 条计算指标;视频另从预测轨迹中选择代表样例渲染 |
|
| 86 |
+
| 占卡 | 管理器负责训练前安全交接、训练退出后恢复;无需手工停占卡 |
|
| 87 |
+
|
| 88 |
+
**为什么不是按最新日志数字续训?** 只有 checkpoint 中的状态能恢复。原进程没有即时保存接口,本次受控停止后,770084 之后未保存的更新需要重算;三份现存权重 153530、611980、770084 均保留。旧运行显示 `failed`、退出码 143、`termination_signal: 15` 是本次主动停止,并非新的训练故障;占卡恢复已核实。
|
| 89 |
+
|
| 90 |
+
**改变 batch 后,步数不能直接与原来的 917970 比较。** 每步处理更多轨迹,因此完成同样剩余数据需要更少更新;以 `chosen_strategy.json`、`strategy_plan.json` 中的剩余轨迹和新终点为准。不是把原步数照搬再多训一遍,也不是从头初始化。Loss 实现和主要权重保持原项目 v24 不变。
|
| 91 |
+
|
| 92 |
+
短测各执行 240 次更新,排除前 40 次预热;batch=2/4/8/16 的有效窗口吞吐约为 38.6/73.4/143.9/287.3 个/秒。约 7.4 倍是短测计算吞吐,不是整轮训练耗时承诺;正式阶段还包括数据加载、长上下文和数值评估。
|
| 93 |
+
|
| 94 |
+
10 月 6 日 18:34 的首批正式训练检查:GPU 利用率约 47%(每 5 秒采样,范围 27%–54%),显存峰值约 27.4 GiB;这是短时间实测,不代表之后所有阶段。原 loss 与混音不改,后续继续用数值评估判断拟合质量。
|
| 95 |
+
|
| 96 |
+
305990、459330 的权重在此次工作前已被“两份保留”规则轮换删除,不能补齐这两个保存点的 60 条;原 12 条报告仍保留,不把缺少的结果记为零或冒充已完成。
|
| 97 |
+
|
| 98 |
+
临时目录由程序自动设置,不需要你手工 `export`:普通临时文件及预测视频渲染中间文件放在 `HFD_lzk/tmp`;仅进程通信 socket 放在本机 `/tmp/hfd-lzk-ipc-…/pymp-…`,正常退出时清理本次通信子目录,异常强杀可能残留。具体路径写入实验的 `input_identity.json`;占卡程序原有锁文件不改动。
|
| 99 |
+
|
| 100 |
+
## 1 重新进入环境并指定本次续训
|
| 101 |
+
|
| 102 |
+
打开终端,复制:
|
| 103 |
+
|
| 104 |
+
```bash
|
| 105 |
+
cd /cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/HFD_lzk
|
| 106 |
+
source .venv-psi/bin/activate
|
| 107 |
+
export PYTHONPATH="$PWD/src:$PWD"
|
| 108 |
+
export CUDA_VISIBLE_DEVICES=3
|
| 109 |
+
export HFD_NPROC_PER_NODE=1
|
| 110 |
+
export MANIFEST="$PWD/data/hfd_third_party_multi_v5/training_pilot_300/selection.json"
|
| 111 |
+
export HFD_TRAIN_RUN="$PWD/runs/third_party_parallel_continue_20261006_1828"
|
| 112 |
+
```
|
| 113 |
+
|
| 114 |
+
这些命令只设置当前终端的路径,**不会重复启动训练或停止占卡**。重开终端后重新执行即可;查看当前任务时不要用时间戳另造目录名。
|
| 115 |
+
|
| 116 |
+
本次仍是固定的 300 条 third-party(六类各 50 条),不重新筛选、不生成数据,不接入其他任务或视频输入。
|
| 117 |
+
|
| 118 |
+
## 2 查看运���状态和恢复进度
|
| 119 |
+
|
| 120 |
+
### 第一步 查看状态
|
| 121 |
+
|
| 122 |
+
```bash
|
| 123 |
+
python scripts/train/third_party_run.py status \
|
| 124 |
+
--run-dir "${HFD_TRAIN_RUN:?请先执行第1节设置环境}"
|
| 125 |
+
```
|
| 126 |
+
|
| 127 |
+
`${HFD_TRAIN_RUN:?...}` 会在变量为空时直接提醒,避免再次误查项目根目录的 `run_status.json`。路径报错不代表训练文件被删除。
|
| 128 |
+
|
| 129 |
+
| 输出 | 怎么理解 |
|
| 130 |
+
|---|---|
|
| 131 |
+
| `training`,但 `launcher_alive: false` | 管理进程已退出,状态是遗留记录,不会自动续跑;原实验就是这种情况 |
|
| 132 |
+
| `training` 且 `launcher_alive: true` | 工作流在运行;具体是测试、补评估还是训练,要看下面的 `workflow_state.json` |
|
| 133 |
+
| `completed` 且 `training_returncode: 0` | 训练正常完成;再确认 `keeper_restore_verified: true` |
|
| 134 |
+
| `failed` | 查看日志中的具体错误,不能当成正常结束 |
|
| 135 |
+
|
| 136 |
+
重启或强制中断时,进程可能来不及写回最终状态。不要手工把 `training` 改成 `completed`,这不会恢复模型。
|
| 137 |
+
|
| 138 |
+
### 第二步 查看正在做哪一步
|
| 139 |
+
|
| 140 |
+
```bash
|
| 141 |
+
python -m json.tool "${HFD_TRAIN_RUN:?请先执行第1节设置环境}/workflow_state.json"
|
| 142 |
+
```
|
| 143 |
+
|
| 144 |
+
| 阶段 | 含义 |
|
| 145 |
+
|---|---|
|
| 146 |
+
| `benchmark_inference` | 比较 1/2/4 个进程的真实推理速度 |
|
| 147 |
+
| `benchmark_training` | 比较 batch=2/4/8/16;此处产生的参数更新全部丢弃,不覆盖正式模型 |
|
| 148 |
+
| `historical_metrics` | 补齐历史保存点的 60 条指标,复用原来 12 条,再计算其余 48 条 |
|
| 149 |
+
| `continuing_training` | 已进入正式续训;保存点评估时会暂时不更新 loss,查 `events.jsonl` 区分 |
|
| 150 |
+
| `completed` | 训练与数值评估结束,还要检查管理器退出码和占卡恢复 |
|
| 151 |
+
|
| 152 |
+
正式续训开始后,`events.jsonl` 中会有 `checkpoint_resume_passed` 和 `batch_migration_started`,记录恢复步数、所选 batch、新预算和学习率。仅看到启动 PID,不等于这些步骤已通过。
|
| 153 |
+
|
| 154 |
+
本次复查表写入 `$HFD_TRAIN_RUN/strategy_review.md`,训练曲线写入 `review.md` 和 `curves.svg`。这些文件会在对应阶段完成后出现;旧报告和视频留在原目录。
|
| 155 |
+
|
| 156 |
+
### 如果再次中断
|
| 157 |
+
|
| 158 |
+
先查状态和日志,把报错交给助手。保留整个 checkpoint 和其中的 `strategy_state.json`,不要只复制模型。新策略改变了分组和学习率映射,不能套用原 batch=2 的续训命令,也不要重新从 770084 开始覆盖进度。
|
| 159 |
+
|
| 160 |
+
启动器负责 GPU 3 身份核验、占卡交接和退出恢复。不需要手工执行占卡停止命令;不要用 `pkill -f monitor.py`、`pkill python` 或旧 PID 强行清卡。
|
| 161 |
+
|
| 162 |
+
| 沿用的实验设置 | 数值 |
|
| 163 |
+
|---|---|
|
| 164 |
+
| 数据/显卡 | 300 条,纯音频,1 张 H200 |
|
| 165 |
+
| batch | 16;不足 16 条的小批按实际条数训练,不丢弃或重复填充 |
|
| 166 |
+
| 训练预算 | 保留已完成部分,只完成第 3 遍剩余 148 条 |
|
| 167 |
+
| 保存模型 | 剩余轨迹完成约一半及全部完成时,在完整轨迹边界保存;本轮新权重不按旧两份规则删除 |
|
| 168 |
+
| 自动预测 | 中途 60 条、最终 300 条,均无视频渲染;历史可用保存点补至 60 条 |
|
| 169 |
+
|
| 170 |
+
一条 session 会对应很多次密集更新,不是一步训练。增大 batch 保持数据覆盖,但改变了梯度平均方式,不等于逐位复现旧训练;性能与拟合质量都要检查。
|
| 171 |
+
|
| 172 |
+
## 3. 中间结果自动记录 之后统一查看
|
| 173 |
+
|
| 174 |
+
不需要你守在电脑前。本次新增记录都在第 1 节设置的 `$HFD_TRAIN_RUN` 下,程序自动写入磁盘;关闭查看日志的终端不影响后台训练。
|
| 175 |
+
|
| 176 |
+
| 文件/目录 | 自动记录什么 |
|
| 177 |
+
|---|---|
|
| 178 |
+
| `strategy_review.md` | 历史和新保存点的指标对比、完整报告链接 |
|
| 179 |
+
| `review.md` | 本轮训练曲线与事件记录 |
|
| 180 |
+
| `curves.svg` | loss、控制准确率、辅助损失、学习率和梯度曲线 |
|
| 181 |
+
| `metrics.jsonl` | 开头 50 步逐步记录;之后每 20 步及实际语义更新点记录,保存稀疏监督指标 |
|
| 182 |
+
| `strategy_origin.json` 链接的原目录 `history/step_*/report.json` | 已完成的历史保存点 60 条指标及六类/语言/音量分组,不重复计算或复制音频 |
|
| 183 |
+
| `metrics_evaluations/step_*/report.json` | 新保存点的 60 条/最终 300 条指标,预测轨迹在 samples 子目录 |
|
| 184 |
+
| `events.jsonl`、`train.log` | 保存/预测完成与失败事件、完整控制台日志 |
|
| 185 |
+
| `run_status.json` | 后台任务状态、退出码、占卡恢复是否通过 |
|
| 186 |
+
| `checkpoints/` | 新模型、优化器、学习率及策略数据位置等完整恢复文件 |
|
| 187 |
+
| `resume_identity.json` | 本次从哪里恢复、恢复步数、数据位置和原训练预算 |
|
| 188 |
+
| `training_benchmarks.json`、`inference_benchmarks.json` | 真实吞吐、显存、推理并行度对比 |
|
| 189 |
+
| `gpu_usage.jsonl` | 每 5 秒记录 GPU 3 利用率、显存、功耗与流程阶段;随本次管理器退出而停止 |
|
| 190 |
+
| `chosen_strategy.json`、`strategy_plan.json`、`strategy_state.json` | 选中的策略、完整剩余轨迹计划、当前已完成轨迹 |
|
| 191 |
+
|
| 192 |
+
曲线约每 5,000 步更新。数值评估使用原混音和完整 50 Hz 轨迹,模型自己生成控制与后续动作,只有初始姿态取自数据。推理进程各自保存自己的语音历史,不混用其他 session;不更新模型参数。
|
| 193 |
+
|
| 194 |
+
保存点数值评估期间仍会暂停参数更新,但不再等待 CPU 视频渲染。检查对应目录的 `worker_*.log` 或已完成的 `samples/*/result.json`,不要只因 loss 暂停就判断卡死。旧 12 条检查约 52 分钟,其中约 48 分钟用于渲染和封装;新流程已将这部分移出中途检查。
|
| 195 |
+
|
| 196 |
+
本轮采用本地记录,不依赖 WandB 登录、不上传音频或视频。先看 `strategy_review.md` 和训练曲线;最终再通过代表样例视频检查动作自然度。
|
| 197 |
+
|
| 198 |
+
想知道是否还在跑,只需:
|
| 199 |
+
|
| 200 |
+
```bash
|
| 201 |
+
python scripts/train/third_party_run.py status \
|
| 202 |
+
--run-dir "${HFD_TRAIN_RUN:?请先执行第1节设置环境}"
|
| 203 |
+
```
|
| 204 |
+
|
| 205 |
+
想临时看日志:
|
| 206 |
+
|
| 207 |
+
```bash
|
| 208 |
+
tail -n 50 -f "${HFD_TRAIN_RUN:?请先执行第1节设置环境}/train.log"
|
| 209 |
+
```
|
| 210 |
+
|
| 211 |
+
此时 Ctrl-C 只退出日志查看。**若确实要停止训练**,确认变量指向要停止的实验,再执行:
|
| 212 |
+
|
| 213 |
+
```bash
|
| 214 |
+
python scripts/train/third_party_run.py stop \
|
| 215 |
+
--run-dir "${HFD_TRAIN_RUN:?请先设置要停止的实验目录}"
|
| 216 |
+
```
|
| 217 |
+
|
| 218 |
+
停止可能丢失最近保存点后的进度;不要使用 `kill -9` 或 `pkill python`。正常完成、一般报错或通过上述 stop 退出后,管理器会尝试恢复 GPU 3 占卡;检查 `keeper_restore_verified: true` 才算恢复成功。训练期间该字段为 `null` 是尚未进入恢复阶段。强杀、断电或整机关闭无法保证自动恢复。
|
| 219 |
+
|
| 220 |
+
## 4 查看已经保存的训练效果
|
| 221 |
+
|
| 222 |
+
先看本次 `strategy_review.md` 对比三个历史保存点的 60 条指标;原目录的 12 条视频仍可查看。新保存点评估完成后也会加入汇总。最终还要确认 `phase: completed`、`training_returncode: 0` 和 `keeper_restore_verified: true`;性能短测通过不代表抗干扰能力已经学会。
|
| 223 |
+
|
| 224 |
+
| 问题 | 看什么 |
|
| 225 |
+
|---|---|
|
| 226 |
+
| 能否听懂真实指令 | 控制决策和真实预测动作是否改善 |
|
| 227 |
+
| 能否忽略旁话 | 背景出现时是否误启动、误打断;指令重叠时是否漏响应 |
|
| 228 |
+
| 动作是否合理 | 是否明显抖动、漂移、反复启动,或一直不动 |
|
| 229 |
+
|
| 230 |
+
新的固定 60 条包含原来的 12 条,六类各 10 条,中英各 30 条,高音量 14 条(约 23%,接近训练集 24%)。汇总同时提供逐条平均和有效帧加权的背景误触发率及分母,避免少量背景帧左右结论。最终 300 条保留这 60 条的推理种子,并单列其结果,方便与历史保存点比较;不能直接拿 300 条均值与 60 条均值判断涨跌。784 条保留集仍不使用。
|
| 231 |
+
|
| 232 |
+
简要原理:混合音频经过 Qwen 提取带历史的语义,控制头决定等待/听指令/执行,时长头和动作头生成机器人动作。背景改变,但正确动作标签不变,以此学习忽略第三方人声。不输入图片、视频或 VQA;渲染视频只是输出。本次实验最初保留 Qwen 预训练底座、新初始化动作/控制等模块,没有加载师兄 HFD checkpoint。本次续训恢复的是我们自己已训练的权重和优化器,不是再次新初始化。
|
AGENTS.md
ADDED
|
@@ -0,0 +1,29 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# HFD_lzk isolated-worktree rules
|
| 2 |
+
|
| 3 |
+
This directory is an isolated handoff copy for LZK. It is not the owner's
|
| 4 |
+
canonical HFD repository.
|
| 5 |
+
|
| 6 |
+
1. Never edit, delete, move, or run output-producing commands against the
|
| 7 |
+
sibling canonical repository at
|
| 8 |
+
`/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/HFD`.
|
| 9 |
+
2. This local repository intentionally has no Git remote and its pre-push hook
|
| 10 |
+
blocks every push. Do not add a remote, remove the hook, or push this work to
|
| 11 |
+
`yediong/Humanoid-Full-Duplex` unless the owner explicitly reviews and
|
| 12 |
+
authorizes that operation.
|
| 13 |
+
3. Local commits and branches are encouraged. Name new branches with the
|
| 14 |
+
`lzk-YYMMDDHHMM-...` pattern so experiments remain traceable.
|
| 15 |
+
4. After changing source code, configuration, scripts, tests, or documentation,
|
| 16 |
+
add a concise Chinese entry to `CHANGELOG.md`, run focused checks, and make a
|
| 17 |
+
local focused commit.
|
| 18 |
+
5. Never commit `runs/`, checkpoints, datasets, runtime indexes, caches,
|
| 19 |
+
credentials, `.env` files, SSH keys, rendered videos, or virtual
|
| 20 |
+
environments.
|
| 21 |
+
6. `.venv-psi` is a shared read-only environment link. Do not run `pip install`,
|
| 22 |
+
`pip uninstall`, or otherwise modify that environment. If dependencies must
|
| 23 |
+
change, create a separate environment owned by HFD_lzk and set `PSI_VENV`.
|
| 24 |
+
7. The datasets and public model paths documented in
|
| 25 |
+
`260904-lzk交接文档.md` are shared inputs. Treat them as read-only. New data
|
| 26 |
+
generation must use a new LZK-owned output directory.
|
| 27 |
+
8. Training and inference consume shared GPUs and the repository training
|
| 28 |
+
launcher manages the shared GPU occupancy monitor. Coordinate GPU use with
|
| 29 |
+
the owner before starting a job.
|
CHANGELOG.md
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
DIALOG.md
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
EXPERIMENT.md
ADDED
|
@@ -0,0 +1,394 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
## 2608101515 — Qwen 流式语义与原生 KV 能力验证
|
| 2 |
+
|
| 3 |
+
### 要回答的问题
|
| 4 |
+
|
| 5 |
+
当前闭环动作效果不理想,可能有两种不同根因:
|
| 6 |
+
|
| 7 |
+
1. Qwen 在 0.6 秒小窗口中根本没有正确理解语音,导致后面的动作模型无语义可用。
|
| 8 |
+
2. Qwen/KV 中已经保留了正确语义,但动作模型没有稳定学会把这份语义映射成对应动作。
|
| 9 |
+
|
| 10 |
+
本实验只做旁路诊断,不修改训练、推理或 checkpoint,不把解码文本写回正式 KV,也不运行 flow 动作生成。
|
| 11 |
+
|
| 12 |
+
### 实验对象与生产对齐
|
| 13 |
+
|
| 14 |
+
- Run:`runs/posttrain_full_duplex/hfd.gpus4.2608101424`
|
| 15 |
+
- Checkpoint:`ckpt_19881`,包含当前 action/control、Qwen 最后两层与 final norm 的训练权重。
|
| 16 |
+
- 输入严格复用正式链路:16 kHz PCM16、0.6 秒窗口、0.4 秒步长、生产 `streaming_source_prefix_text()`、Qwen 原生 `DynamicCache`,只在完整 source 边界清空。
|
| 17 |
+
- 语音使用生产 TTS 原文件和生产 assignment 中的增益,叠加到正式数据池中的连续安静 HVAC 或真实办公室背景;同一 source 从头到尾只使用一条连续背景。
|
| 18 |
+
- 六类指令专门覆盖容易混淆的方向与复合语义:向前走并抓取、身前双手挥动、向前走、向右走、向左走、向后走。
|
| 19 |
+
- 每类使用全部八个音色(四个英文、四个中文),共 48 条单轮样本。每条先输入两个纯背景窗口,再输入完整混合语音,随后再输入一个连续纯背景窗口。
|
| 20 |
+
- 多轮部分使用八条三轮 session,共 24 个 turn;每个 session 的指令顺序轮换,轮间保留 20/30 个背景窗口,整个 session 不刷新 KV。这样可以排除“类别固定等于第几个 turn”的位置泄漏。
|
| 21 |
+
|
| 22 |
+
### 评价与负对照
|
| 23 |
+
|
| 24 |
+
- 对 Qwen 最后一个 history-conditioned audio-EOS 表示做跨音色留一的余弦最近类中心分类;测试音色从未参与其类中心计算。
|
| 25 |
+
- 另做英训中测、中训英测和“最近的其他音色是否属于同一指令”检索。
|
| 26 |
+
- 负对照包括:语音前纯背景、只给最后 0.6 秒且清空历史 KV、以及 200 次按音色独立打乱标签。
|
| 27 |
+
- 旁路文本解码只在完整流式 KV 后追加一次只读问题,检查 Qwen 能否直接复述最新动作指令。它证明语义是否存在,但不等价于 action head 已经会使用该语义。
|
| 28 |
+
|
| 29 |
+
### 结果
|
| 30 |
+
|
| 31 |
+
| 条件 | 跨音色类中心 | 英→中 / 中→英 | 其他音色最近邻 | 随机机会 |
|
| 32 |
+
| --- | ---: | ---: | ---: | ---: |
|
| 33 |
+
| 完整 KV,语音最后一窗 | 37.5% | 20.8% / 29.2% | 45.8% | 16.7% |
|
| 34 |
+
| 完整 KV,再过一个背景窗 | 41.7% | 20.8% / 12.5% | 56.3% | 16.7% |
|
| 35 |
+
| 只看最后一窗、无历史 KV | 20.8% | 25.0% / 16.7% | 20.8% | 16.7% |
|
| 36 |
+
| 语音前纯背景 | 16.7% | 16.7% / 16.7% | 16.7% | 16.7% |
|
| 37 |
+
|
| 38 |
+
- 完整 KV 两项跨音色准确率都超过对应打乱标签分布的 95 分位(25.0% 和 27.1%);背景负对照精确落在随机水平。
|
| 39 |
+
- 单轮旁路文本解码 6/6 正确:分别得到“抓取某物”“挥动双手”“Walk forward”“向右走路”“向左走”“向后走路”。这直接说明当前 Qwen 能听懂抓取、挥手和四个方向,而不是完全没有语义能力。
|
| 40 |
+
- 去掉历史 KV 后,跨音色分类从 37.5% 降到 20.8%,接近随机;说明 0.6 秒窗口确实不能独立承担完整指令,持续 KV 是必要的,而且当前实现确实在利用历史。
|
| 41 |
+
- 多轮原始表示的简单跨音色类中心准确率只有 25.0%(三类随机为 33.3%),不能认为它已经形成稳定、音色无关的“最新指令向量”。多轮最后一句直接复述为 6/8 正确;两次失败都复述成同一 session 较早出现的“抓取”,说明长 KV 中语义仍在,但“当前最新指令”与旧指令的绑定有时会混淆。
|
| 42 |
+
|
| 43 |
+
### 结论
|
| 44 |
+
|
| 45 |
+
1. **单轮效果差不能主要归因于 Qwen 完全听不懂。** 当前 checkpoint 的 Qwen/KV 能正确复述六类测试指令,方向和抓取语义都存在。单轮更可疑的瓶颈是“Qwen hidden → action”的监督与映射,而不是 ASR 级理解彻底失败。
|
| 46 |
+
2. **KV 不能关,也不能只看当前小窗口。** 无 KV 的最后一窗已接近随机;当前 0.4 秒持续追加原生 KV 的方向是对的。
|
| 47 |
+
3. **多轮仍有真实语义风险。** Qwen 没有整体失忆,但会在少数音色下把旧指令当成最新指令。后续若增加语义辅助监督,重点应约束“当前 turn 的最新意图”,而不是只证明历史里存在某条指令。
|
| 48 |
+
4. **旁路文本监督有价值,但不应把生成文本接回正式推理链。** 更安全的方案是在训练时仅于语义更新/每轮语音结束处增加一个轻量的最新指令文本或语义对齐 loss,标签可直接来自现有 metadata;推理仍走原来的 Qwen hidden/KV → action,不增加文本生成延迟,也不改变全双工缓存合同。
|
| 49 |
+
|
| 50 |
+
### 音色策略审计
|
| 51 |
+
|
| 52 |
+
- 用户提出的策略是合理的:一次 source 内固定音色,下一次遇到同一 motion 时再换音色,可以同时保证时序稳定��扩大同一动作的音色覆盖。
|
| 53 |
+
- 当前 `fixed_voice_per_record` 只保证**同一个 primitive turn 内**固定;同一条 motion record 每次重访仍永远使用同一个音色,并不会随 epoch 轮换。
|
| 54 |
+
- 对正式训练划分只读审计:99,457 条 primitive record 组成 33,172 条多轮 session,其中只有 1,511 条从头到尾保持同一音色,31,661 条会在 turn 间换人,整段同音色比例仅 4.56%。如果部署语义是同一个用户连续说多轮,这也是一处值得修正的训推分布差异。
|
| 55 |
+
- 推荐后续改为:每个完整 session 先确定一个音色/语言身份,所有 turn 固定使用它;同一 motion 在下一 epoch 或下一次采样时按可复现规则轮换。该规则必须同时提供给长度分桶 sampler,并随 checkpoint 恢复 epoch/采样状态,否则会破坏长度估计、八卡负载均衡和完美续训。
|
| 56 |
+
|
| 57 |
+
### 局限与清理
|
| 58 |
+
|
| 59 |
+
- 本次只覆盖六类指令、八个音色、两类正式背景和一个 checkpoint;结论足以排除“Qwen 完全听不懂”,但不能替代全指令集评估。
|
| 60 |
+
- 余弦类中心只是最低成本探针;Qwen 原始 hidden 不是专门训练的语义 embedding,因此低线性可分性不等于绝对没有信息,文本解码结果需结合判断。
|
| 61 |
+
- 多轮探针刻意让一个 session 保持同一说话人,符合目标部署,但与当前大多数训练 session 会换音色的现状不同;该差异已作为独立发现记录。
|
| 62 |
+
- 临时脚本、JSON、Python cache 和中间表示在记录数值后全部删除;没有保留 WAV、hidden、模型副本、渲染结果或额外 WandB run。正式代码没有合入任何实验开关。
|
| 63 |
+
|
| 64 |
+
## 2608101718 — 真实推理 action condition 并行语义解码
|
| 65 |
+
|
| 66 |
+
### 为什么重做
|
| 67 |
+
|
| 68 |
+
`2608101515` 的跨音色类中心数值只能回答“原始 hidden 是否含有高于随机的语义信号”,不能当成 Qwen 的听懂准确率。Qwen 的 audio hidden 从未被训练成按动作类别聚类的 embedding,因此 `37.5%/41.7%` 看起来低并不等于只听懂四成;上一实验的多轮复述也没有证明被检查的状态就是动作模型实际消费的状态。本实验直接插入正式多轮推理调用链,在不改正式代码和 KV 的前提下,从动作生成当场使用的 condition 分叉做只读文本解码。
|
| 69 |
+
|
| 70 |
+
### 与正式推理严格对齐的实现
|
| 71 |
+
|
| 72 |
+
- Run/权重:`runs/posttrain_full_duplex/hfd.gpus4.2608101424/checkpoints/ckpt_19881`。
|
| 73 |
+
- 直接调用正式 `scripts/infer/full_duplex_audio_to_motion.py`,使用正式三轮场景:“向前走并抓取某物”→“双手在身前挥动”→“向左走路”,连续办公室背景、5 个初始 idle 窗口、25/30 个轮间背景窗口、每轮 450 帧动作、10-step flow solver 均不改。
|
| 74 |
+
- 在每轮语音结束处截获 `encode_stream_input` 返回的同一个 `FullDuplexStreamCondition`。该 condition 的 `[1,18,2048]` 当前 Qwen hidden 随后确实被 `forward_action_conditioned` 的全部 10 次 flow 求解调用使用。
|
| 75 |
+
- 文本旁路先深复制当时的 Qwen `DynamicCache`,再追加一个文本问题:“只复述实时音频中最近听到的最后一条机器人动作指令”。使用 checkpoint 自带的 Qwen LM head 贪心生成;同时 teacher-force 三个候选答案并计算平均 token log-prob,避免只凭一个贪心字符串下结论。
|
| 76 |
+
- 每次旁路前后校验正式 KV 的长度、attention、首层 K/V 形状和数值签名完全不变;解码文本不写回动作 condition、正式 KV 或后续推理。
|
| 77 |
+
- 负对照使用完全相同的正式音频窗口和 prompt,但在每轮开始前清空一份影子 KV,只保留当前轮语音。它用于区分“当前语音本身听不懂”和“长历史把新旧指令混淆”。
|
| 78 |
+
|
| 79 |
+
### 结果
|
| 80 |
+
|
| 81 |
+
| 轮次 | 目标 | 完整生产 KV 解码 / 候选最优 | 仅当前轮 KV 解码 / 候选最优 |
|
| 82 |
+
| --- | --- | --- | --- |
|
| 83 |
+
| 1 | 向前走并抓取 | 向前走并抓取 / 抓取 | 向前走并抓取 / 抓取 |
|
| 84 |
+
| 2 | 双手挥动 | **向前走并抓取 / 抓取** | 双手挥动 / 挥动 |
|
| 85 |
+
| 3 | 向左走 | **向前走并抓取 / 抓取** | 向左走 / 向左 |
|
| 86 |
+
|
| 87 |
+
- 第二轮完整 KV 的平均 token log-prob:抓取 `-0.246`、挥动 `-0.409`、向左 `-4.966`;清空旧轮历史后变为抓取 `-4.969`、挥动 `-0.186`、向左 `-6.877`。
|
| 88 |
+
- 第三轮完整 KV:抓取 `-0.245`、挥动 `-0.664`、向左 `-1.677`;仅当前轮时变为抓取 `-3.985`、挥动 `-4.228`、向左 `-0.737`。
|
| 89 |
+
- 完整 rollout 共更新 Qwen/KV 101 次、生成 2020 帧;三次被检查的正式 condition 均由 10 次 action flow 调用消费,旁路检查均确认生产 KV 未改变。
|
| 90 |
+
|
| 91 |
+
### 结论与边界
|
| 92 |
+
|
| 93 |
+
1. `37.5%/41.7%` 不是分类准确率意义上的“听懂率”,只能作为高于负对照的弱证据;以后不再单独用它判断 Qwen 能力。
|
| 94 |
+
2. 当前三条语音单独都能被同一个 Qwen 正确复述,说明第二、三条输入本身有效;但完整连续 KV 在后两轮强烈偏向第一条“抓取”,证明存在真实的“旧指令抢占最新��令”问题。
|
| 95 |
+
3. 该问题发生在动作模型实际使用的同一 condition 上,不是另造输入得到的旁路假象;但文本 LM head 与 action head 的读出方式不同,因此它定位的是 Qwen/KV 的最新意图绑定风险,不能单独等价为动作一定输出第一条指令。
|
| 96 |
+
4. 本轮只做诊断,不把文本生成并入正式推理。后续若做训练优化,应在每轮语音边界加入“最新指令”辅助监督或显式 turn/recency 表示,而不是把慢速文本解码串进实时动作链。
|
| 97 |
+
|
| 98 |
+
### 数据侧同步发现与验证
|
| 99 |
+
|
| 100 |
+
- 正式语义版本 17 将同一 session 的全部 2~4 轮固定为同一个说话人和语言,消除旧数据中约 `95.45%` session 中途换人的伪分布。
|
| 101 |
+
- 训练集建立两个静态 voice/session variant:每条 primitive motion 在一个 epoch 内恰好出现两次,使用两个不同音色,并分别进入独立随机邻居组合;验证集仍只出现一次。这样数据量是 `2×`,而不是激进扩成八音色的 `8×`。
|
| 102 |
+
- 全量真实清单审计:train 为 `99,457` 条 primitive、`198,914` 次使用、`66,241` 条 session;所有 primitive 都恰好两次且两个音色不同,混音色 session 为 `0`。val 为 `11,389` 条 primitive、`3,790` 条 session,每条恰好一次,混音色 session 同样为 `0`;没有长度/context/background 过滤导致漏样本。
|
| 103 |
+
- variant 在 dataset 初始化时静态确定并写入 checkpoint 训练/推理合同,不依赖 persistent DataLoader worker 收不到的动态 epoch 状态,因此长度分桶、DDP 对齐和完美续训可复现。旧语义版本 16 的数据和合同保持不变。
|
| 104 |
+
|
| 105 |
+
### 清理
|
| 106 |
+
|
| 107 |
+
实验脚本、JSON、日志、动作 CSV/NPY、临时音频和 Python cache 在记录上述结果后删除;未创建额外 WandB run,未修改旧 run/checkpoint,正式代码中没有保留文本解码 hook。
|
| 108 |
+
|
| 109 |
+
## 2608101857 — 最新指令文本监督标签与训练旁路验证
|
| 110 |
+
|
| 111 |
+
### 目的与约束
|
| 112 |
+
|
| 113 |
+
上一项真实生产-KV 探针表明,多轮历史中的旧指令可能压过最新指令。本实验验证能否在不改变实时推理、不引入 VAD、也不污染生产 KV 的前提下,于每轮语音结束后的因果 `listen → motion` 边界增加一次“最新指令精确文本”辅助监督。
|
| 114 |
+
|
| 115 |
+
### 标签来源审计
|
| 116 |
+
|
| 117 |
+
- TTS manifest:`/cpfs01/users/buwendong/code/qwen3-tts/metadata.jsonl`,共 142,220 行、5,220 个唯一音频目录。
|
| 118 |
+
- TTS 生成脚本明确使用 `content_short_description` 生成 `*_en.wav`,使用 `content_short_description_zh` 生成 `*_zh.wav`;因此目标文本按当前 session 实际选中的音色后缀选择,而不是把中文语音错误监督成英文。
|
| 119 |
+
- 142,220 个 filename 均能在 Seed motion metadata 中找到;英文原文仅有 214 条尾部空格差异,执行 `.strip()` 后为 `142,220 / 142,220` 精确一致。同一音频目录不存在冲突的中英文文本对。
|
| 120 |
+
- 5,220 个目录的八种音色 WAV 共 `41,760 / 41,760` 全部存在。示例:`000000` 为 `checking whole body / 检查全身`,`000001` 为 `standing with arms crossed / 双臂交叉站立`,`000002` 为 `from drinking while standing to standing / 从站立饮酒到站立`。
|
| 121 |
+
- 这里证明的是“WAV 的生成输入文本、训练标签和 motion 配对标签”全量一致;它不能替代逐个 WAV 的人工听检或 100% 无误的声学 ASR。此前生产音频旁路已抽样确认中英文真实音频可被 Qwen 解码,但不把抽样结果冒充全部 41,760 个文件的听写质检。
|
| 122 |
+
|
| 123 |
+
### 实现与训推边界
|
| 124 |
+
|
| 125 |
+
- 每轮只在当前音频已进入真实 Qwen KV、并发生 `listen → motion` 控制切换的同一个 Qwen 更新点监督一次;idle、listen 中间帧和 dense action 复用步不产生文本 loss。
|
| 126 |
+
- 将当场的 Qwen `DynamicCache` 逐层按有效 batch 行复制到一次性旁路,追加固定查询和精确答案做 teacher forcing。旁路结束即释放,下一窗口继续读取原生产 KV;查询和答案不会进入后续动作历史。
|
| 127 |
+
- 冻结 Qwen LM head,只训练项目原本已经开放的 Qwen 尾部两层/final norm。loss 按八卡全局有效文本 token 数取标准均值,避免不同语言 token 长度或各 rank 边界数不同造成权重偏差。
|
| 128 |
+
- 用正式 Qwen tokenizer/processor 全量审计:固定查询为 18 token,最长双语目标为 33 token,EOS 为 1 token;首个 0.6 秒窗口占 82 token,后续每窗 17 token,模型上下文为 32,768。版本 18 将 source 上限从 1,923 收紧到 1,919 窗,最坏情况下仍为旁路保留 80 token,覆盖 52-token 查询/答案且不裁剪生产 KV。
|
| 129 |
+
- 正式推理没有文本输入、标签或 decoder 调用;合同明确记录 `vad=false`、`inference_text_decoder=false`。本版本只实现这一项辅助 loss,没有同时加入 VAD、显式 turn marker 或双记忆结构。
|
| 130 |
+
|
| 131 |
+
### 验证结果
|
| 132 |
+
|
| 133 |
+
- 单测覆盖:中英文标签选择、每轮唯一因果目标、TextOp 变换透传、teacher-forcing mask、旁路 KV 不增长、Qwen 尾层有梯度而冻结 LM head 无���度、v18 合同与 Shell 默认值。专项结果为 `100 passed, 1 skipped`,完整 `tests/` 为 `118 passed, 1 skipped`;仓库根目录的额外 OpenPI/JAX 测试因该虚拟环境未安装可选 `jax` 而无法收集,与本次 HFD 路径无关。
|
| 134 |
+
- 真实 dataset 抽样:8 条记录可构造 2 轮 session,目标数恰为 2,文本边界与 Qwen 更新/时长边界完全重合;四卡首批 source 的目标分别出现在约第 120、160、240、140 个 dense step,证明短测确实越过了监督点。
|
| 135 |
+
- 四卡真实训练:batch 1/卡、16 条 source、motion 限 1 step、最多 260 optimizer step;四个 rank 均越过首个文本边界,完整前向、辅助反向、DDP 同步和 checkpoint 保存成功。稳定后约 13.5 step/s,峰值显存约 21.1 GiB/卡,无 OOM、NaN 或死锁。
|
| 136 |
+
- 短测目录约 13 GiB,验证 checkpoint 合同后已安全删除;没有创建 WandB 垃圾曲线,GPU 占卡守护已恢复。
|
| 137 |
+
|
| 138 |
+
### 结论
|
| 139 |
+
|
| 140 |
+
数据标签和训练旁路满足预期,可以进入正式 v18 对照训练。它直接增强“当前完整 KV → 最新指令文本”的可读性,但是否能改善 grab、方向和多轮动作仍必须由相同步数的闭环推理比较判断;本短测只证明实现正确和可训练,不预先宣称最终动作质量提升。
|
| 141 |
+
|
| 142 |
+
## 2608121705 — interruption-v1 四卡真实训推 smoke
|
| 143 |
+
|
| 144 |
+
### 目的
|
| 145 |
+
|
| 146 |
+
验证语义版本 20 不仅能通过合成单测,还能在真实 Qwen2.5-Omni、真实连续音频、真实动作模型和四卡 DDP 下完成训练、保存、加载及四种闭环推理。该实验只判断工程链路和训推合同,不用 3 step 随机权重评价动作质量。
|
| 147 |
+
|
| 148 |
+
### 训练配置与结果
|
| 149 |
+
|
| 150 |
+
- 本机 4×H200,固定 50 条过拟合 source,每卡 2 条并行 source,最多 3 个 optimizer step;保留真实 0.6 秒 waveform、Qwen 原生 KV、39D 动作、30/20/10 RTC 和全部正式 loss,仅把 motion 调试上限设为 20 个语义步并关闭 WandB,避免生成无意义曲线。
|
| 151 |
+
- 启动时确认 train/eval 各 50 条且 primitive overlap 为 0;真实 Qwen2.5-Omni-3B、约 5 亿参数 Action Transformer、control/duration/text 辅助路径均成功初始化。
|
| 152 |
+
- 三步 loss 约为 `140 → 124 → 117`,稳定约 `4.0 s/step`、`96.5 active windows/s`,峰值显存约 `20.8 GiB/GPU`;四个 rank 完成反向、梯度平均并在合法 source 边界保存 `ckpt_3`,marker 含完整 v20 interruption 合同。
|
| 153 |
+
|
| 154 |
+
### 四路并行推理
|
| 155 |
+
|
| 156 |
+
- 用四张 GPU 同时运行 turn-taking/train、turn-taking/val、interruption/train、interruption/val;每张空闲卡从同一队列领取一个 case。
|
| 157 |
+
- 为缩短纯链路 smoke,临时设置 2-step flow、10 fps 渲染,并忽略随机 3-step 模型的 control 预测;生产默认仍是 hard control、10-step flow、50 fps。
|
| 158 |
+
- 四个 case 均完成 rollout、动作文件、视频渲染、完整连续音轨合成和 `.complete` marker。耗时约为 interruption/train `27.3 s`、interruption/val `40.1 s`、turn-taking/train `40.5 s`、turn-taking/val `53.7 s`。
|
| 159 |
+
|
| 160 |
+
### smoke 发现并修复的问题
|
| 161 |
+
|
| 162 |
+
1. v20 合同最初错误继承了 `interruption=false`;已改为真实值并加入合同回归。
|
| 163 |
+
2. 两种行为的 action 指标最初读取外层遗留的整条 source `steps`,而不是当前窗口的 `parallel_steps`,可能导致 shape 错误或模式归类错误;已改成当前 batch 元数据并加入生产路径回归。
|
| 164 |
+
3. 历史 Bash 样本名因 `echo | tr` 带有尾随下划线,Python 队列最初找不到实际 `.complete`;调度器现在严格复现已有命名,并用测试锁定兼容行为。
|
| 165 |
+
|
| 166 |
+
### 清理与结论
|
| 167 |
+
|
| 168 |
+
两次 smoke 的临时 run、checkpoint、视频、音频和 cache 均按明确临时根目录清理,没有写入正式 `runs`,也没有创建 WandB run;GPU 占卡程序已恢复。实验确认 v20 的真实训练、checkpoint 合同和四路动态推理链路可以运行,但收敛质量仍必须由正式八卡长训及固定闭环视频判断。
|
| 169 |
+
|
| 170 |
+
## 2608140306 — v22 时长概率分布与五窗口累积四卡 smoke
|
| 171 |
+
|
| 172 |
+
### 目的与配置
|
| 173 |
+
|
| 174 |
+
- 验证语义版本 22 的纯 Qwen 语义控制、48 桶对数时长概率分布、五个相邻 stride-1 窗口平均更新以及 `75/18/7` 任务混合能在真实四卡 DDP 中训练、保存并严格加载。
|
| 175 |
+
- 使用 4×H200、每卡 batch 1、512 条受限 primitive、motion 调试上限 10 个语义步、60 次 optimizer update、关闭 WandB;所有 Qwen waveform/KV、39D 动作、30/20/10 RTC 和正式 loss 路径均保留。
|
| 176 |
+
|
| 177 |
+
### 结果
|
| 178 |
+
|
| 179 |
+
- train/validation 构造分别得到 913/226 条受限 session,比例接近 `75/18/7` 且 source 隔离通过。60 次更新正常结束,稳定 source compute 约 `0.32~0.33 s/update`、约 `60~63 dense windows/s`,峰值显存约 `21.1 GiB/GPU`;验证、best checkpoint 和完整恢复状态均保存成功。
|
| 180 |
+
- checkpoint 合同确认:语义版本 22、纯语义 control、48 桶 `0.5~180` 秒时长分布、五窗口累积、task variants `4/1/1/0`。正式三轮闭环完成连续 KV、2,020 帧动作、���频渲染和音轨合成。
|
| 181 |
+
- 随机短训模型没有自然进入 motion,因此另用只改变控制执行策略的链路探针强制走动作分支;模型从概率分布中解码并冻结中位数为 `420` 原始帧(`8.4` 秒),随后完成 200 帧生成、5 fps 快速渲染和音轨合成。该探针只验证工程数据流,不评价 60-step 权重的动作质量。
|
| 182 |
+
- 专项回归最终为 `125 passed, 1 skipped`;Python 编译、Shell/JSON 语法和 `git diff --check` 均通过。没有建立 WandB 曲线。
|
| 183 |
+
|
| 184 |
+
### 结论与清理
|
| 185 |
+
|
| 186 |
+
v22 训练、DDP、checkpoint、时长分布解码和完整推理链路均可运行;短 smoke 不提供收敛结论。所有本次临时 run、checkpoint、渲染和失败探针目录在记录结果后统一清理,占卡程序恢复。
|
| 187 |
+
|
| 188 |
+
## 2608170028 — 单指令单动作数据消融准备与 v22 终点评估
|
| 189 |
+
|
| 190 |
+
### 目的
|
| 191 |
+
|
| 192 |
+
- 检查同一句指令对应大量不同 mocap 轨迹是否让动作目标互相平均,尤其影响“`双手在身前挥动`”等长期学不好的动作。
|
| 193 |
+
- 只准备可独立启动的 A/B 数据消融,不替换正式训练数据;同时复核 `hfd.gpus8.2608141550` 的 80 万步终点是否正常。
|
| 194 |
+
|
| 195 |
+
### 筛选方法与数据量
|
| 196 |
+
|
| 197 |
+
- 在正式多轮安全过滤一致的 `110,846` 条候选上,按精确英文和中文 TTS 文本分组,共得到 `4,246` 组;当前数据中英文文本本身恰好也是 `4,246` 个唯一值。
|
| 198 |
+
- 每组优先非镜像原始动作,先取最接近组内中位时长的 8 个候选,再读取真实 50 Hz G1 轨迹,以关节 jerk、根节点加速度、朝向角加速度、结束速度和运动强度代表性联合排序。该方法避免只选最短、最慢或文件名最靠前的动作。
|
| 199 |
+
- 原始候选 `110,846 → 4,246`,删除 `106,600`、保留率 `3.8305%`。以正式 v22 配置真实重建后,train 为 `3,828` primitive、`6,823` session;val 为 `418` primitive、`179` session。对照原正式数据分别为 train `99,457/176,563`、val `11,389/5,042`。
|
| 200 |
+
- 中文侧另有 `123` 个完全相同的句子对应多个不同英文命令,典型是快/慢/开始被翻成同一句。消融版对这些记录只保留无歧义英文音色,避免中文仍然一对多;没有删除 WAV 或 motion。
|
| 201 |
+
- “`wave two hands in front / 双手在身前挥动`”共有 4 条(2 条原始、2 条镜像)。选择结果为 `wave_two_hands_R_001__A533`;其关节 jerk p99 为 `0.0950`、关节加速度 RMS 约 `0.00812`,均优于 002 的约 `0.1399/0.01169`。
|
| 202 |
+
|
| 203 |
+
### 80 万步正式 run 终点
|
| 204 |
+
|
| 205 |
+
- Run:`runs/posttrain_full_duplex/hfd.gpus8.2608141550`,WandB run `t91ohnza`,正常完成 `800,000` step,无 NaN、OOM 或中途退出。
|
| 206 |
+
- 终点 train/eval loss 分别约 `0.963/1.612`,eval action loss `0.654`,eval control accuracy `87.90%`,其中 turn-taking `87.60%`、interruption `90.81%`;最新指令 token accuracy `81.72%`,时长 MAE 约 `3.38 s`。
|
| 207 |
+
- 固定闭环中,train turn-taking 的三轮状态顺序正确;val turn-taking 在背景段出现一次短暂 motion 误触发。interruption 能进入 listen 并切换新动作,但 val 仍有一次 `idle → motion → listen` 绕路。
|
| 208 |
+
- Stop train 样例能停止并响应后续新指令;Stop val 样例在约 `8.0 s` 进入 idle 后,于约 `12.8 s` 在没有新语音时错误恢复 motion,持续到约 `18.0 s`。因此逐窗口 control accuracy 较高仍不能证明 Stop 已稳定保持。
|
| 209 |
+
|
| 210 |
+
### 结论与实验边界
|
| 211 |
+
|
| 212 |
+
1. 单指令单动作确实彻底去除了同一英文指令的 motion 目标冲突,也为双手挥动留下了更平滑轨迹;但它同时砍掉约 `96%` primitive,多样性损失很大,必须和完整数据在相同训练预算下 A/B,不能直接宣布更好。
|
| 213 |
+
2. 提高 Stop 比例可以增加急停边界和停止后背景监督,预计会降低错误频率;但 80 万步已经反复看到大量 Stop 样本,终点仍会被旧 motion 语义重新触发,因此只加比例不等于获得了“当前有效意图已经清空”的持久记忆。
|
| 214 |
+
3. 硬编码一个 Stop 专用锁只是最低成本诊断;更可扩展的根本方向是持久交互意图状态:普通命令建立/替换 active intent,Stop 清空它,新有效命令再建立它。该抽象可同时覆盖 cancel/wait/resume/interruption,而不是每增加一个任务再写一条规则。本轮不改模型,先新增 `eval/stop_false_motion_reactivation_rate` 量化问题。
|
| 215 |
+
4. 消融清单位于共享数据区 `manifests/canonical_motion_per_instruction_v1.json`,约 3.1 MiB;只含选择结果和质量统计。未创建 checkpoint、WandB run、渲染或大型临时文件。
|
| 216 |
+
|
| 217 |
+
## 2608180305 — Backchannel 文献设计、数据审计与四卡链路验证
|
| 218 |
+
|
| 219 |
+
### 文献结论与任务边界
|
| 220 |
+
|
| 221 |
+
- [InteractSpeech(Findings of EMNLP 2025)](https://aclanthology.org/2025.findings-emnlp.424/)把 backchannel 定义为对方仍持有话轮时的短促关注/认同反馈,并使用文本场景生成、统一 TTS、精确时间戳和专家过滤构造全双工交互数据。对 HFD 的直接启发是:短反��应作为动作中的重叠输入,不能另起一个 `listen→motion` 回合。
|
| 222 |
+
- [Yeah, Un, Oh(NAACL 2025)](https://aclanthology.org/2025.naacl-long.367/)把 backchannel 分为 continuer 与 assessment,并在连续、非平衡真实对话上逐帧判断时机和类型;其数据中 continuer 明显多于 assessment。HFD 因此覆盖“嗯哼/是的”等继续信号与“哦/明白/哇”等轻量评价,而不是只做一个关键词表。
|
| 223 |
+
- [Easy Turn](https://arxiv.org/abs/2509.23938)把 complete、incomplete、backchannel、wait 作为不同流式状态,并从短于 2 秒的候选出发做扩写、重标注、TTS 和 ASR 质检。HFD 同样把 WAV 限制在 2 秒内并做 train/validation 短语隔离,但本任务的目标不是让机器人输出 backchannel,而是让机器人正确忽略用户的非抢话反馈。
|
| 224 |
+
- [DuplexSLA](https://arxiv.org/abs/2605.20755)及 [Full-Duplex-Bench v1.5](https://arxiv.org/abs/2507.23159)都将 interruption 与 listener backchannel 分开评价:新意图应改变当前行为,而 backchannel 应保持正在进行的输出。这正是本次 supervision 的正反边界。
|
| 225 |
+
|
| 226 |
+
### 短语与音频
|
| 227 |
+
|
| 228 |
+
- 共 18 组双语含义。train 独占 12 组:8 组 continuer(`嗯嗯/Mm-hmm`、`嗯/Yeah`、`啊哈/Uh-huh`、`对/Right`、`是啊/Yep`、`好/Okay`、`嗯哼/Mhm`、`是的/Yes`)和 4 组 assessment(`明白/I see`、`好的/Got it`、`没错/Exactly`、`哦/Oh`)。validation 独占另外 6 组:`嗯,是的/Yeah, right`、`对的/That's right`、`确实/Indeed`、`嗯,没错/Mm, exactly`、`原来如此/That makes sense`、`哇/Wow`。
|
| 229 |
+
- `Yeah/Right/对/好` 在孤立情况下有歧义,保留它们是刻意的 hard negative:模型必须结合“当前动作仍在进行、没有新动作谓词”的完整 KV 判断,而不能把任何人声都当成 interruption。第一版不加入带明确命令、疑问或新命题的长句,避免标签本身含糊。
|
| 230 |
+
- 使用 Qwen3-TTS VoiceDesign 和项目既有 8 个 voice prompt 生成 `144` 个 WAV。审计结果:train/val=`96/48`,continuer/assessment=`96/48`,时长 `0.24~1.60 s`,全部为 PCM16、16 kHz、单声道,SHA256 和帧数检查无错误。数据位于 `/cpfs01/videogen/yetinghong/datasets/hfd_backchannel_commands_v1`,不纳入 Git。
|
| 231 |
+
|
| 232 |
+
### 真实数据构造与规模
|
| 233 |
+
|
| 234 |
+
- 一条 backchannel session 仍由 2~4 个真实 motion primitive 组成,只在其中一个正在执行的 motion 内叠加一次短反馈。插入时刻为该 motion 开始后的 `1.2~6.0 s`,对齐 20 帧/0.4 秒 Qwen 更新格;完整语音被听完后仍留至少 `1.2 s` 真实 motion。
|
| 235 |
+
- 普通指令与短反馈使用同一 session voice;两者叠加到同一段连续办公室背景,不拼接背景、不清 KV、不重置 RTC、future progress、motion duration 或最新真实指令标签。短反馈覆盖的逐帧 control GT 仍为 `</motion>`。
|
| 236 |
+
- 最长 0.1% 清理后的正式规模:train `220,703` 条 session,其中 turn-taking/interruption/stop/backchannel=`132,423/33,105/22,070/33,105`;validation `6,302` 条,对应 `3,782/945/630/945`。比例约为 `60/15/10/15`,且原三个任务的绝对数据量都没有减少。
|
| 237 |
+
|
| 238 |
+
### GT 与工程验证
|
| 239 |
+
|
| 240 |
+
- 真实 `getitem` 的 4 条带音频审核视频位于 `runs/backchannel_gt_review_260818`,分别覆盖 train/val 与 continuer/assessment。每个 `backchannel_review_clip` 约 10~11 秒,短反馈固定在裁剪后第 4 秒;视觉抽帧及 source metadata 均确认动作时间轴没有被短反馈切换成 listen/idle。
|
| 241 |
+
- 专项测试结果为 `133 passed, 1 skipped`。4×H200 真实训练 smoke 使用完整 Qwen2.5-Omni、约 5 亿参数动作模型和四卡 DDP,完成 2 次反向、8 条 validation、`ckpt_2` 与 best checkpoint;eval loss `108.534348`、峰值约 `20.8 GiB/GPU`,checkpoint 明确保存 `control_target=motion_unchanged`、`qwen_kv=append_normally_without_reset` 和 `rtc_and_future_progress=unchanged` 合同。
|
| 242 |
+
- 本实验只证明数据、训练、验证和 checkpoint 链路正确;backchannel 闭环能力是否收敛,仍需正式八卡训练后看 `eval/backchannel_motion_preservation_rate`、`eval/backchannel_false_interruption_rate` 及固定 train/val 视频,不能由 2 step smoke 预先下结论。
|
| 243 |
+
|
| 244 |
+
## 2608200117 — 纯语义控制输入四类可分性验证
|
| 245 |
+
|
| 246 |
+
### 问题和人话版实验
|
| 247 |
+
|
| 248 |
+
当模型在 Stop 和 backchannel 之间摆动时,先要分清两种根因:Qwen 交给控制头的信息已经把几种话听混了,还是信息里其实分得开,只是正式控制头没有在更广的时间轴上稳定用好。本实验就像把模型当时“脑子里看到的信息”拿出来,只训一个很小的分类器,看它能否分清:新动作指令、Stop、短 backchannel、没人说话的背景。
|
| 249 |
+
|
| 250 |
+
### 与正式链路的对齐
|
| 251 |
+
|
| 252 |
+
- 使用正在训练的 `runs/posttrain_full_duplex/hfd.gpus8.2608180017` 与不可变 `ckpt_617428`;只读 checkpoint,不修改 run、WandB 或任何正式输出。
|
| 253 |
+
- 当前语义版本 22 的 `state_aware_serial_control=false`,所�� `condition.memory_token[:, 0]` 就是 `control_head` 真正收到的 2048 维 Qwen audio-EOS 向量,没有混入 state、RTC、future progress 或上一个 token。
|
| 254 |
+
- 64 个样本由四类×8 个正式音色×2 种说法/条件构成。每个样本都先在同一条连续安静 HVAC 背景中听到同一条前置动作指令,然后才输入新动作指令、`Stop/Stop now`、`Mm-hmm/Indeed` 或等长背景;16 kHz waveform、0.6 秒窗口、0.4 秒步长、TTS 增益、PCM16 回量化、Qwen 原生 KV 和只在 source 边界清空均与正式链路一致。
|
| 255 |
+
- 为了不把“语音还没听完/模型仍处于 listen”误当语义判断,四类都在完整语音窗口后再输入 4 个背景更新(1.6 秒)才读取向量和控制输出。
|
| 256 |
+
- 主指标是“留一个完整音色”:小分类器用其他 7 个音色学习,再测从没给它看过的第 8 个音色。另外整组留出其中一种说法,并做 20 次随机标签负对照。
|
| 257 |
+
|
| 258 |
+
### 结果
|
| 259 |
+
|
| 260 |
+
| 指标 | 结果 |
|
| 261 |
+
| --- | ---: |
|
| 262 |
+
| 四类随机机会 | 25.0% |
|
| 263 |
+
| 留一完整音色线性可分率 | 87.5% |
|
| 264 |
+
| 留一整组说法线性可分率 | 71.875% |
|
| 265 |
+
| 随机标签均值 / 95 分位 | 23.125% / 39.141% |
|
| 266 |
+
|
| 267 |
+
- 跨音色混淆矩阵按 `action/stop/backchannel/background` 排列:`action=15/16`、`stop=10/16`、`backchannel=15/16`、`background=16/16`。Stop 特征的跨音色线性分界相对最弱,但仍明显高于随机对照。
|
| 268 |
+
- 不另训小分类器,直接查当前正式控制头:新动作指令均预测 motion `16/16`(平均概率 93.33%),Stop 预测 idle `15/16`(89.35%),backchannel 预测 motion `16/16`(93.62%),纯背景预测 motion `16/16`(99.85%)。
|
| 269 |
+
|
| 270 |
+
### 结论和边界
|
| 271 |
+
|
| 272 |
+
1. **不支持“Qwen 已经把四类语义全听混”这一根因。** 控制头收到的真实向量可以跨过未见音色分出四类,并且正式控制头在这批实例上只错一个 Stop。
|
| 273 |
+
2. **不能由此宣布整个控制问题已解决。** 本次只有每类两种说法,而且只读一个语音后的固定时刻;它不代表数十秒闭环中每个窗口都不会摆动。现有训练中的周期性摆动更应优先检查更广短语/背景覆盖、关键事件相对权重和长时间持续性,而不是先改 Qwen 架构。
|
| 274 |
+
3. 本诊断不生成动作、视频、新 checkpoint 或 WandB run。工具只在内存中保留特征,输出 JSON 不包含 2048 维特征本身;本次 `/tmp` 结果在数值入库后清理。
|
| 275 |
+
|
| 276 |
+
## 2608250439 — 动态 source 补位学习率与数据预取四卡 A/B
|
| 277 |
+
|
| 278 |
+
### 实验问题与设计
|
| 279 |
+
|
| 280 |
+
- 问题一:动态补位按设计提高有效槽位后,为什么早期效果反而比固定槽差;重点排查是否是每个 optimizer step 变得更“实”后,旧 `1e-4` 学习率过强,而不是 source 换槽污染了 Qwen KV。
|
| 281 |
+
- 问题二:source 工作池切换时 GPU 长时间等 `getitem`,能否只调整 CPU 预取而不改变训练样本、顺序和数学。
|
| 282 |
+
- 使用同一份初始权重、正式 train/validation、seed=42、每卡 `B=192`、两卡一组。固定槽 `1e-4` 与动态槽 `1e-4` 同机并发;再以完全相同动态调度补跑 `8e-5` 控制组。动态两组都在第一个完整工作池边界 `step=7067` 评估,构成严格的单变量学习率对照。固定槽首个安全边界为 `step=5696`,只处理两批 lookahead 中较短的一批,因此只作参考,不冒充同曝光量的严格对照。
|
| 283 |
+
|
| 284 |
+
### 首次完整 validation
|
| 285 |
+
|
| 286 |
+
| 调度 / 峰值 LR | eval step | 总 loss | action loss | control loss | control acc | 最新指令 token acc | 时长 MAE |
|
| 287 |
+
| --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: |
|
| 288 |
+
| 固定槽 / `1e-4` | 5,696 | 2.4503 | 0.9024 | 0.7440 | 76.06% | 81.78% | 3.468 s |
|
| 289 |
+
| 动态槽 / `1e-4` | 7,067 | 3.8414 | 0.9466 | 1.5810 | 60.77% | 81.84% | 4.009 s |
|
| 290 |
+
| 动态槽 / `8e-5` | 7,067 | 3.4900 | 0.8795 | 1.3033 | 62.90% | 81.90% | 3.965 s |
|
| 291 |
+
|
| 292 |
+
- 严格动态对照中,`8e-5` 令总/action/control loss 分别改善约 `9.1%/7.1%/17.6%`,control accuracy 提升 `2.13` 个百分点;两组最新指令 token accuracy 几乎相同。动态槽动作 loss 已略优于固定槽,说明换槽后的独立 mask、逻辑 RoPE 和 source-local KV 没有破坏动作/语义链路。
|
| 293 |
+
- 动态调度平均槽位填充率约 `94.79%`,固定槽约 `82.93%`。动态组到首次安全评估累计约 `257.3` 万个有效 source-window 行,固定组约 `181.4` 万;所以固定槽与动态槽的首个安全评估并非相同数据曝光,且固定槽先看到排序后的短 source。动态槽剩余的 control 差距不能全部归因于学习率,也不能仅凭该早期 checkpoint 宣布最终动作质量优劣。
|
| 294 |
+
- 动态 `1e-4/8e-5` 的 action grad norm 均值约为 `5.11/4.95`,95 分位约 `7.67/7.53`;降学习率主要改善参数实际更新幅度,而不是消除数据本身的困难窗口。新正式训练默认使用 `8e-5`,但不增加控制头专用规则或额外 loss。
|
| 295 |
+
|
| 296 |
+
### 数据等待优化与边界
|
| 297 |
+
|
| 298 |
+
- 原先精确 lookahead 需要两批 source,却只让 worker 队列提前准备一批;GPU 排空当前工作池后,另一批只能同步等待。现在预取容量恰好覆盖下一组两批:正式 `B=192/workers=16` 的 factor 为 `24`。这只改变数据何时准备好,不改变 DataLoader sampler 的产出顺序或 `getitem` 的确定性 RNG。
|
| 299 |
+
- 三组新实验在冷启动后的普通 source 边界等待约 `17~26 s`;旧正式八卡 factor=12 的可见边界通常约 `100~200 s`。该对比受 GPU 数、共享盘竞争和缓存影响,只证明现场等待明显收窄,不将比例外推为稳定集群基准。冷首批仍约 `308~357 s`。
|
| 300 |
+
- 音色查找建表的 70 万次独立微基准约 `13.46 s → 0.238 s`,只代表该热点,不代表整个首批能加速 56 倍。未采用 `in_order=False`、先完成先训练、增加任务乱序或更多 worker,因此训练质量不因 I/O 优化改变。
|
| 301 |
+
- 修复 W&B 等待指标:同一次取 source 的等待仅记在该工作池第一个 optimizer event,后续 dense step 记零;旧曲线中一条等待被重复数千次的问题不再误导诊断。
|
| 302 |
+
|
| 303 |
+
### 工程验证
|
| 304 |
+
|
| 305 |
+
- 专项回归:`150 passed, 1 skipped`;Python 编译、训练 shell 语法及 `git diff --check` 通过。
|
| 306 |
+
- 三个 A/B run 只用于诊断;结果入库后清理 `/tmp` checkpoint,不并入正式 runs。占卡脚本另做“2/3 忙、0/1 空闲,再释放 2/3”的受控测试,能从 `0,1` 自动稳定扩展到 `0,1,2,3`,且训练卡仅瞬时低利用率时不会被误抢。
|
| 307 |
+
|
| 308 |
+
## 2608260041 — Hesitation/Pause 数据生产与真实 getitem GT 审核
|
| 309 |
+
|
| 310 |
+
### 任务定义与文献依据
|
| 311 |
+
|
| 312 |
+
- [Full-Duplex-Bench](https://arxiv.org/html/2503.04721) 将说话人仍持有话轮的句中 pause 与 turn ending 分开,受控停顿主要落在约 `0.4~1.0 s`;[DuplexSLA](https://arxiv.org/html/2605.20755) 同样要求模型在用户停顿思考时继续等待,而不是把静音直接判成话轮结束。HFD 因此把 `pause` 定义为“一条尚未说完的指令内部出现纯背景停顿”,不是动作完成后的 terminal idle。
|
| 313 |
+
- [Full-Duplex-Bench v3](https://arxiv.org/html/2604.04847) 将 filler、hesitation、false start 和 self-correction 分开评估;[Teaching BERT to Wait](https://aclanthology.org/2022.naacl-main.60/)及[Disfluency Generation](https://aclanthology.org/2023.findings-acl.728/)说明,不完整上下文应显式训练“等待更多证据”,且合成不流畅语料可用于补足真实稀缺事件。HFD 将 hesitation 分成 `60%` 同意图 filled-pause 和 `40%` 改变意图 self-repair。
|
| 314 |
+
- 全部新样本只增加数据,不增加特殊 token 或推理硬规则。pause/hesitation 从不完整前缀开始进入 `</listen>`,停顿、填充词、改口词和最终指令期间持续 listen;只有完整最终意图经过既有因果延迟后才进入 `</motion>`。Qwen KV 在整个 2~4 轮 session 中持续写入,办公室背景、说话人、RTC 和动作状态都不因不流畅事件重置。
|
| 315 |
+
|
| 316 |
+
### 音频构造与生产审计
|
| 317 |
+
|
| 318 |
+
- 没有直接裁剪原 WAV。工具先在英文词边界和中文 jieba 词边界拆分 metadata 指令,再用 Qwen3-TTS 对前缀、后缀重新合成;self-repair 使用“另一条动作指令的不完整前缀+改口词+完整最终指令”,最终动作及最新指令文本监督只对应修正后的指令。同一 session 固定一个语言和一个正式音色,所有片段继续叠加同一条连续办公室背景。
|
| 319 |
+
- 静默停顿基础采样为 `0.4~2.8 s`、对齐 `0.4 s` Qwen 网格,短/中/长三组概率为 `60/30/10`。若填充词或改口词本身更长,则透明地将有效 gap 扩展到能够完整容纳该词并保留背景边缘,绝不裁断人声。train/validation 使用不同 phrase ID;一个 session 最多插入一个 pause 或 hesitation 事件,避免一次加入过多变量。
|
| 320 |
+
- 冻结生产计划覆盖 `5,104` 条可安全拆分的双语指令,拒绝 `116` 条不适合自然分词的指令。最终生成 `81,664` 个指令片段和 `80` 个 filler/repair marker,共 `81,744` 个 WAV;manifest SHA256 为 `e1011f31d09e8240af77536f5756d9e049f056de5fba8cb9c3f0e290311f64fd`。
|
| 321 |
+
- 全量并行审计确认 `81,744/81,744` 均为 PCM16、16 kHz、单声道,帧数、SHA256、有限值和峰值合法。生产过程中自动重试 13 个超过 20 秒的 TTS 跑偏音频,并额外重生成 74 个超过 8 秒的指令片段和 2 个超过 2.64 秒的 marker。最终指令片段时长 `p50=1.52 s, p95=3.28 s, p99=3.92 s, max=7.92 s`;filled marker 最大 `1.92 s`,repair cue 最大 `2.461 s`,质量上限违规为 0。
|
| 322 |
+
- 完成标记 `reports/production_complete.json` 绑定 manifest 格式、SHA、行数和 8 音色。训练启动只验证完成标记和 manifest,选中某条 WAV 时再做逐文件核验,避免 8 个 DDP rank 在启动阶段重复打开超过 65 万个共享盘小文件;这只减少元数据 I/O,不改变样本或训练数学。
|
| 323 |
+
|
| 324 |
+
### 正式数据规模
|
| 325 |
+
|
| 326 |
+
| split | sources | stride-1 windows | source 时长 |
|
| 327 |
+
| --- | ---: | ---: | ---: |
|
| 328 |
+
| train | 267,792 | 576,896,608 | 3,204.98 h |
|
| 329 |
+
| validation | 7,680 | 16,553,100 | 91.96 h |
|
| 330 |
+
|
| 331 |
+
- train source 数:turn-taking `132,423`、interruption `33,105`、Stop `22,070`、backchannel `33,105`、hesitation `27,756`、pause `19,333`。对应实际窗口占比为 `47.915/7.443/10.441/13.443/12.454/8.304%`。
|
| 332 |
+
- validation source 数:`3,782/945/630/945/819/559`,窗口占比为 `48.016/7.414/10.532/13.333/12.438/8.267%`。train/validation primitive split 在合成前已隔离,不存在跨 split source 重用。
|
| 333 |
+
- hesitation train 中 filled/self-repair 为 `16,632/11,124`,约 `60/40`;pause 为 `19,333`。新任务先按中英指令去重和时长分层,再限制每条完整指令 train 最多 32、validation 最多 8 条 motion;旧语义版本 22 的四个任务选择结果和绝对 source 数逐条保持不变。
|
| 334 |
+
- 八卡、每卡 `B=192` 时全局有 1,536 个并发 source 槽;以全部 dense windows 粗略折算,一个理想满槽 epoch 约 `576,896,608 / 1,536 = 375,584` 个 optimizer step。动态补位、source 边界和 DDP 对齐会使日志中的实际 epoch 边界略有差异,因此该数字只用于预算,不冒充精确 wall-clock 预测。
|
| 335 |
+
|
| 336 |
+
### 指标、GT 和工程验证
|
| 337 |
+
|
| 338 |
+
- 新增四个直接闭环时机指标:`pause_premature_motion_rate` / `hesitation_premature_motion_rate` 统计最终意图完整前是否误动;`pause_motion_timing_mae_seconds` 统计实际进入 motion 与 GT 边界的秒级误差;`hesitation_final_over_provisional_accuracy` 检查改口期间没有执行 provisional 意图且最终边界正确进入 motion。另保留两类任务的 action/control loss 和 control accuracy。最后一项是控制时机门控指标,不等价于动作语义本身完全正确,仍需结合最新指令文本准确率和闭环视频。
|
| 339 |
+
- 从正式语义版本 23 的真实 `getitem` 还原 36D GT、逐帧控制标签以及重叠 0.6 秒 Qwen 窗口的完整连续音频,并以 50 fps 渲染六条带音频审核视频。覆盖 silent-pause、filled-pause、self-repair 各自 train/validation;停顿为 `1.6~2.8 s`,最终指令完整后 `1.04~1.36 s` 才进入 motion。视频、音频和 metadata 位于共享数据根 `hfd_hesitation_pause_v1/gt_review/2608260018`,不纳入 Git。
|
| 340 |
+
- 六条视频的 H.264/AAC 容器、16 kHz 音轨和动作时长一致;抽帧复核未见脚穿地、世界朝向突跳或拼接断裂。专项回归为 `156 passed, 1 skipped`,覆盖旧 v22 选择逐条不变、新任务合同、连续 KV、数据构造、trainer/eval 指标、推理套件和故障注入;Python、Shell、JSON 和差异检查通过。
|
| 341 |
+
- 本阶段只完成数据与工程可用性验证。正式四卡训练和自动闭环推理必须等用户听看六条 GT 并确认时间轴质量后启动;GT 审核不能替代长期收敛实验。
|
| 342 |
+
|
| 343 |
+
## 2608260247 — 静止段与犹豫改口质量修复审计
|
| 344 |
+
|
| 345 |
+
### 异常手部动作根因
|
| 346 |
+
|
| 347 |
+
- 第一条 v23 GT 中第二轮语音前的抬手不是音画错位。旧 `_sample_stationary_sequence("idle")` 会把当前机器人 pose 平滑插值到随机 idle 素材的绝对 pose,再插值回来;素材和当前状态虽然分别静止,但腕部角度不同,因此构造器凭空生成一次与任何指令无关的动作。
|
| 348 |
+
- 旧构造的审核 idle 段手腕最大逐帧速度约 `0.068~0.070 rad/frame`,原始安静素材自身约 `0.0013~0.0018`。v24 将素材转换为相对首帧残差并叠加到当前 anchor,根朝向保持当前值;新六条 GT 的对应值多为 `0.0017~0.0047`,最差 terminal idle 为 `0.0104`,没有旧版数量级的伪手势。
|
| 349 |
+
- 六个任务的 session 都通过同一 idle 采样函数,所以 turn-taking、interruption、emergency-stop、backchannel、hesitation、pause 均受旧问题影响,也均由同一处修复。idle 专用审计进一步从 68 条素材中保留 65 条和 `3,876` 个安全 anchor,排除 `neutral_idle_loop_002__A120`、`003__A120`、`004__A120`;listen 的表达性手势池不使用该严格阈值。
|
| 350 |
+
|
| 351 |
+
### 犹豫音与改口覆盖
|
| 352 |
+
|
| 353 |
+
- v2 覆盖层复用 v1 的 `81,664` 个已审计指令片段,只生成 80 个新 marker。28 个 `嗯/呃/唔/Um/Uh/Er/Hmm` 音色化样本时长为 `0.800~2.160 s`,中位数 `1.000 s`、均值 `1.092 s`;短语式 filler 共 12 个,实际 `0.800~1.440 s`、中位数 `0.960 s`。第三条审核视频中的 `呃……` 实际持续 `(42,711-12,845)/16,000 = 1.867 s`,已不是旧版的短促语气词。
|
| 354 |
+
- self-repair 按约 `65/35` 混合显式 cue 与隐式 restart。后者是“错误意图的不完整前缀+0.4~2.8 s 连续背景+直接完整重说最终指令”,没有“改成/I mean”提示词。完整 train 计数为显式 `7,257`、隐式 `3,806`,validation 为 `217/126`。
|
| 355 |
+
- v2 manifest SHA256 为 `f96071d4dbb59e8035544beea3f192b583b569cff019e8a54681b7ca012ca17d`;`81,744/81,744` 行通过格式、时长、哈希和音频数值审计,0 个无效文件。
|
| 356 |
+
|
| 357 |
+
### 数据、GT 与四卡工程验证
|
| 358 |
+
|
| 359 |
+
- 正式构造为 train `267,850` session / `577,086,270` stride-1 窗口,validation `7,698` session / `16,519,542` 窗口。train 六类 session 数依次为 turn-taking `132,423`、interruption `33,105`、Stop `22,070`、backchannel `33,105`、hesitation `27,873`、pause `19,274`。
|
| 360 |
+
- 六条真实 `getitem` 带音频 GT 位于 `hfd_hesitation_pause_v2/gt_review/2608260247`。第一条第二轮前 `7.8~9.0 s` 的逐帧抽样确认机器人在 idle 内保持自然站立,进入第二轮 listen 后才开始听取手势;第三条为拖长 filler;第五条保留显式改口,第六条为没有提示词的隐式 restart。
|
| 361 |
+
- 专项测试为 `160 passed, 1 skipped`。四卡完整正式数据 smoke:每卡 `B=1`、60 step;首个 Qwen/DeepSpeed/共享盘冷启动 step 约 140 s,之后稳态约 `12 step/s`、`53 active windows/s`,`data_wait=0`、峰值 `21.1 GiB/GPU`,模型、optimizer、scheduler、sampler 与 source-boundary marker 全部保存。
|
| 362 |
+
- `ckpt_60` 随后完成两轮隐式改口短推理:43 次连续 cache update、860 帧有限输出;渲染得到 5 fps H.264 视频与 AAC 音轨,总时长 17.4 s。期间发现外部 SONIC renderer 的默认 XML 仍指向迁移前路径,HFD 已改为显式传入新 GMR XML,并修复 render/mux 失败仍误标 `.complete` 的错误。
|
| 363 |
+
|
| 364 |
+
### 结论边界
|
| 365 |
+
|
| 366 |
+
- 本轮证明 v24 数据构造、完整四卡训练、checkpoint 和推理/渲染工程链路可运行,并通过真实 GT 排除了已知伪手势;60 step 随机初始化 checkpoint 只用于接口 smoke,不能用其动作质量推断正式收敛效果。正式八卡训练仍需用户听看 v24 六条 GT 后确认。
|
| 367 |
+
|
| 368 |
+
## 2609020131 — 控制结构、历史检索与分阶段训练四卡 A/B 决策
|
| 369 |
+
|
| 370 |
+
### 目的与公平性
|
| 371 |
+
|
| 372 |
+
- 回答三项架构建议是否应进入正式 HFD:用三行 special-token 输出替代 Control MLP、让 action 梯度通过离散 control、以及先训控制再训动作。
|
| 373 |
+
- Control A/B 固定同一生产 checkpoint、seed 42、4 GPU、每卡 `B=192`、1,536 条训练原始记录、同一 64 条验证记录及 2,000 个 stride-1 动作窗口,其中恰有 100 个 Qwen/control 更新点;四个 variant 只改变 control predictor 和 hard/straight-through 路径。
|
| 374 |
+
- 分阶段 A/B 固定相同初始化、数据与总监督量:两边都有 100 次语义更新和 2,000 次动作更新。历史检索探针使用生产 Qwen/KV、连续背景、三轮真实 TTS、6 种排列和 8 个音色,全程不清 KV。
|
| 375 |
+
|
| 376 |
+
### 结果与正式取舍
|
| 377 |
+
|
| 378 |
+
| 设计 | 验证结果 | 决策 |
|
| 379 |
+
| --- | ---: | --- |
|
| 380 |
+
| Control MLP + hard | control accuracy `75.91%` | 保留 |
|
| 381 |
+
| Control MLP + straight-through | `75.94%` | 无实质收益,不采用 |
|
| 382 |
+
| 三行 token + hard / straight-through | `60.32% / 60.29%` | 明显退化,不采用 |
|
| 383 |
+
| 联合训练 | control `75.94%`,action loss `0.62852` | 保留 |
|
| 384 |
+
| 先控制后动作 | control `76.28%`,action loss `0.62795`,计算 `1.279×` | 收益过小,不采用永久两阶段 |
|
| 385 |
+
|
| 386 |
+
- 直接三行输出在数学上只是同一 2,048 维语义向量上的线性三分类;它删掉了现有 Control MLP 的 LayerNorm 和非线性隐层,因此没有因为“叫 special token”就自动更强。纯 control、完全跳过动作网络的复验仍为 MLP `76.28%`、三行输出 `60.33%`,排除了动作开销造成的假差异。
|
| 387 |
+
- 原生产 KV 上,自由解码 latest/previous/first 的 exact accuracy 为 `20.83/4.17/41.67%`,而 teacher-forced token accuracy 为 `84.39/80.18/84.95%`。这证明现有 latest loss 没有擦除历史;真实问题是第一条历史常压过最近轮次,而且逐 token teacher forcing 会高估自由检索能力。
|
| 388 |
+
- 冻结 Qwen 和生产 KV,仅训练三行零初始化 marker,并继续保留完整查询;在未见指令排列上,latest/previous/first 从 `31.25/12.50/81.25%` 变为 `100/75/81.25%`。未见音色子集的 first 有回退,因此这仍是单 seed、小身份 pilot,不能预先宣称正式动作质量提升。
|
| 389 |
+
|
| 390 |
+
### 正式实现边界
|
| 391 |
+
|
| 392 |
+
- 正式版本只采用有正向证据且风险可控的部分:保留 MLP/hard/联合训练,在每个非首轮因果 `listen → motion` 边界增加一条历史文本目标,并用 `latest/previous/first` marker 明确要读取的相对轮次。
|
| 393 |
+
- 两条文本 readout 都从生产 KV 深拷贝一次性 side cache;不会把 query、答案或 decoder 输出写回语音历史。推理不执行文本解码,因此实时速度和动作数据流不变。
|
| 394 |
+
- 数据按 2~4 轮结构安排一条稀疏历史目标:turn 2 读 previous、turn 3 读 first、turn 4 读 previous。正式权重为 latest `0.10`、history `0.05`;DLC 长训需要继续观察历史 token accuracy、最新指令 accuracy、各任务 control 与闭环动作,pilot 不能替代最终结论。
|
FULL_DUPLEX_TASK_RESEARCH.md
ADDED
|
@@ -0,0 +1,120 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# 全双工任务与数据构造调研(260812)
|
| 2 |
+
|
| 3 |
+
## 结论
|
| 4 |
+
|
| 5 |
+
当前最值得迁移到 HFD 的共同做法不是单独增加一个 VAD,而是同时做到三件事:
|
| 6 |
+
|
| 7 |
+
1. 用固定短时间片持续接收环境音,整段 session 保留语义历史;
|
| 8 |
+
2. 显式监督当前应当继续执行、聆听还是空闲;
|
| 9 |
+
3. 通过一条连续时间轴合成 turn-taking、overlap 和 interruption,并用因果标签保证模型不能提前看到未来。
|
| 10 |
+
|
| 11 |
+
因此 interruption-v1 不改变 HFD 的模型主干,而是在现有 turn-taking 数据旁边构造等量的打断 session。新语音先与旧动作重叠;模型仍继续旧动作,直到从当前和历史音频中因果判断应进入 listen,随后平滑接入新动作。
|
| 12 |
+
|
| 13 |
+
## 主流方法
|
| 14 |
+
|
| 15 |
+
### Moshi
|
| 16 |
+
|
| 17 |
+
[Moshi](https://arxiv.org/html/2410.00037) 将用户和系统音频建模为并行流,因此可以持续听、持续说并容纳任意重叠。其训练组合包括约 700 万小时单流语音预训练、约 2,000 小时 Fisher 双通道真实对话、约 170 小时对话 TTS,以及超过 20,000 小时合成口语对话;合成时还加入音量、噪声、混响和 100–500 ms 回声。与音频时间对齐的文本“inner monologue”用于加强语义。
|
| 18 |
+
|
| 19 |
+
对 HFD 的启发:连续音频流与动作流应并行存在;背景音不是“无输入”,而是每个时间片都进入同一个 KV。辅助文本监督可以只在训练时读取真实 KV,不必在部署时先 ASR 再做动作。
|
| 20 |
+
|
| 21 |
+
### SyncLLM
|
| 22 |
+
|
| 23 |
+
[SyncLLM](https://arxiv.org/html/2409.15594v1) 将双人语音离散为约 160–240 ms 的同步块,用说话人和时间标记在单个自回归序列中联合建模两边。论文使用约 193,000 小时合成 SFT、20,000 小时合成非重叠对话及约 2,000 小时 Fisher 真实对话。
|
| 24 |
+
|
| 25 |
+
对 HFD 的启发:训练样本必须有统一物理时钟;语音、token 和动作不能各自拼接后再猜对齐关系。HFD 保持 0.4 s Qwen 更新、50 Hz 动作监督,并在同一帧时间轴上生成标签。
|
| 26 |
+
|
| 27 |
+
### Freeze-Omni
|
| 28 |
+
|
| 29 |
+
[Freeze-Omni](https://arxiv.org/html/2411.00774v5) 冻结语言模型,训练流式语音编码器/解码器;其逐块状态分类器区分继续输入、结束并打断、结束但不打断,并使用约 60,000 条多轮 TTS 数据。
|
| 30 |
+
|
| 31 |
+
对 HFD 的启发:是否打断应当是一个因果状态决策,而不是推理脚本硬编码。HFD 的 `</idle>`、`</listen>`、`</motion>` 控制头承担同样作用,但每个未来动作帧都有自己的标签,避免一个 30 帧 chunk 被错误地当作同一状态。
|
| 32 |
+
|
| 33 |
+
### SoulX-Duplex
|
| 34 |
+
|
| 35 |
+
[SoulX-Duplex](https://arxiv.org/html/2603.14877v1) 以约 160 ms 流式 ASR 和 idle、non-idle、backchannel、complete、incomplete 等状态 token 组织双工行为,并用辅助文本目标加强语义状态;评估关注停止/响应延迟、成功率、误启动和误停止。
|
| 36 |
+
|
| 37 |
+
对 HFD 的启发:除平均动作 loss 外,应分开监控 turn-taking/interruption 动作误差,以及将来加入响应延迟、错误打断、漏打断等闭环指标。
|
| 38 |
+
|
| 39 |
+
### DyaPlex
|
| 40 |
+
|
| 41 |
+
[DyaPlex](https://arxiv.org/abs/2606.03874) 使用冻结的 PersonaPlex 语音塔和因果动作塔,以对齐 RoPE 和 dyadic interleaving 融合约 4,000 小时 Seamless Interaction 数据。其[项目页](https://research.nvidia.com/labs/amri/projects/DyaPlex/)强调交互双方的动作上下文对具身全双工行为很重要。
|
| 42 |
+
|
| 43 |
+
对 HFD 的启发:当前单机器人 HFD 已经保留自身连续状态和 RTC;以后若有用户视频或人体姿态,不应仅提取文本语义,而应把对方动作作为同步的因果输入流。
|
| 44 |
+
|
| 45 |
+
### 任务谱系
|
| 46 |
+
|
| 47 |
+
[全双工语音交互综述](https://arxiv.org/html/2606.19453) 将常见任务概括为普通轮替、零间隔接话、合作式打断、backchannel、第三方说话/噪声、犹豫和长静音、持续重叠等,并建议用 Idle、Listen、Speak、Wait、Dual 等状态描述时间轴。
|
| 48 |
+
|
| 49 |
+
HFD 建议按以下顺序推进:
|
| 50 |
+
|
| 51 |
+
1. 已完成:普通 turn-taking;
|
| 52 |
+
2. 本版:用户在机器人动作中说出新指令,机器人被打断;
|
| 53 |
+
3. 后续:backchannel 与“听到但不执行”的第三方语音/环境人声;
|
| 54 |
+
4. 后续:不完整指令、改口、澄清和误唤醒;
|
| 55 |
+
5. 后续:动作中途口头修正、重复上一步、停止等依赖历史指令的高级任务;
|
| 56 |
+
6. 有对方视觉流后:双人动作同步与持续 overlap。
|
| 57 |
+
|
| 58 |
+
## HFD interruption-v1 数据设计
|
| 59 |
+
|
| 60 |
+
### 1:1 且不扩大训练体量
|
| 61 |
+
|
| 62 |
+
- 训练集原有每条 primitive 在一个 epoch 使用两次、对应两个固定音色变体。
|
| 63 |
+
- v20 将第一次使用构造成 turn-taking,第二次使用构造成 interruption;因此每条 primitive 在两种行为中各出现一次,行为监督严格 1:1,总 primitive 使用次数不增加。
|
| 64 |
+
- 同一 session 内固定同一个说话人/语言;同一 primitive 的两次使用仍选择不同音色,并进入不同邻居组合。
|
| 65 |
+
- validation 对相同 primitive 分别构造 turn-taking 和 interruption,使用同一确定性音色,从而把“��为差异”和“音色差异”分开。
|
| 66 |
+
- sampler 继续按长度组 batch,再随机打乱这些组:两种行为在训练步骤间随机交错,同时避免把长短差距很大的 source 强行塞进同一 GPU batch。
|
| 67 |
+
|
| 68 |
+
### 连续时间轴
|
| 69 |
+
|
| 70 |
+
每条 session 有 2–4 个 turn,2/3 turn 占多数。开头仍是:
|
| 71 |
+
|
| 72 |
+
`随机 0–2.8 s idle → 第一条语音 → 因果 listen → 第一段 motion`
|
| 73 |
+
|
| 74 |
+
普通 turn-taking 路径保持 v19 不变:动作执行完、平滑恢复站立、可选背景间隔,再接下一条语音。
|
| 75 |
+
|
| 76 |
+
interruption 路径为:
|
| 77 |
+
|
| 78 |
+
`旧 motion → 新语音开始且旧 motion 继续 → 因果 listen → 新 motion`
|
| 79 |
+
|
| 80 |
+
- 新语音在旧 motion 开始后 2–6 s 的 0.4 s Qwen 网格上随机出现;
|
| 81 |
+
- 选择起点时保证原始旧动作至少还剩 1.5 s;
|
| 82 |
+
- 语音刚出现不直接使用未来信息切 token。旧 motion 按真实 GT 继续到“检测到有效语音 + 0.4 s 反应时间”的因果边界;
|
| 83 |
+
- 若极端长前导静音超过剩余真实动作,先执行完旧动作,再保持旧动作终态,绝不崩溃、提前读取语义或伪造新动作;
|
| 84 |
+
- 到 listen 边界后,以当前真实姿态为起点,用 1 s 零速度/零加速度 quintic 过渡进入自然 listen;
|
| 85 |
+
- 新 motion 再以该 listen 终态为起点完成统一位置和朝向对齐。
|
| 86 |
+
|
| 87 |
+
### future progress
|
| 88 |
+
|
| 89 |
+
被打断动作的 future progress 始终使用“原始、未截断 motion 的完整帧数”作分母。打断起点不会被当作旧动作的 100%,因此模型在听到下一条语音之前无法从 progress 偷看“马上要被打断”。若旧动作因极端长静音已自然结束,额外终态保持使用完成态 progress。
|
| 90 |
+
|
| 91 |
+
### 语音、背景与缓存
|
| 92 |
+
|
| 93 |
+
- 每个 session 只截取一段连续办公室背景,所有语音都叠加到这条背景上;
|
| 94 |
+
- 语音和旧动作允许重叠,但不同用户指令在 v1 中不互相重叠;
|
| 95 |
+
- 每 0.4 s 将真实 0.6 s 音频窗送入 Qwen,同一 session 从头到尾不清空原生逐层 KV;
|
| 96 |
+
- 动作以 50 Hz stride-1 监督,每次预测 30 帧、执行 20 帧、复用前一窗口预测的后 10 帧 RTC;
|
| 97 |
+
- 只有 source/session 切换才清空 Qwen KV 和 RTC。
|
| 98 |
+
|
| 99 |
+
## 评估设计
|
| 100 |
+
|
| 101 |
+
统一 launcher 对每个 immutable checkpoint 同时跑四项闭环样例:
|
| 102 |
+
|
| 103 |
+
1. turn-taking / train;
|
| 104 |
+
2. turn-taking / val;
|
| 105 |
+
3. interruption / train;
|
| 106 |
+
4. interruption / val。
|
| 107 |
+
|
| 108 |
+
输出固定为:
|
| 109 |
+
|
| 110 |
+
- `<run>/turn-taking/<ckpt>/train/...`
|
| 111 |
+
- `<run>/turn-taking/<ckpt>/val/...`
|
| 112 |
+
- `<run>/interruption/<ckpt>/train/...`
|
| 113 |
+
- `<run>/interruption/<ckpt>/val/...`
|
| 114 |
+
|
| 115 |
+
任务由空闲 GPU 动态领取:4 case/4 GPU 时各跑一个;16 case/4 GPU 时自动排成四批;GPU 完成当前 case 后立即领取下一个。训练侧新增的两条必要曲线是:
|
| 116 |
+
|
| 117 |
+
- `loss_action_turn_taking`:普通轮替窗口的平均动作误差;
|
| 118 |
+
- `loss_action_interruption`:打断窗口的平均动作误差。
|
| 119 |
+
|
| 120 |
+
两条曲线用于判断新能力是否挤压基础能力,不替代闭环视频。下一阶段应增加的首要闭环指标是:新语音起点到 `</listen>` 的响应延迟、误打断率、漏打断率、动作切换处速度/加速度峰值,以及最新指令文本准确率。
|
LICENSE
ADDED
|
@@ -0,0 +1,17 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
Apache License
|
| 2 |
+
Version 2.0, January 2004
|
| 3 |
+
http://www.apache.org/licenses/
|
| 4 |
+
|
| 5 |
+
Copyright 2026 PSI-lab, USC
|
| 6 |
+
|
| 7 |
+
Licensed under the Apache License, Version 2.0 (the "License");
|
| 8 |
+
you may not use this file except in compliance with the License.
|
| 9 |
+
You may obtain a copy of the License at
|
| 10 |
+
|
| 11 |
+
http://www.apache.org/licenses/LICENSE-2.0
|
| 12 |
+
|
| 13 |
+
Unless required by applicable law or agreed to in writing, software
|
| 14 |
+
distributed under the License is distributed on an "AS IS" BASIS,
|
| 15 |
+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
| 16 |
+
See the License for the specific language governing permissions and
|
| 17 |
+
limitations under the License.
|
OMNI_DATA_PREPROCESS.md
ADDED
|
@@ -0,0 +1,128 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Omni 训练数据预处理
|
| 2 |
+
|
| 3 |
+
通用 Omni 缓存入口是 `prepare_omni_training_data.sh`。当前正式全双工
|
| 4 |
+
audio-command(semantics v3)使用连续办公室背景专用入口:
|
| 5 |
+
`prepare_full_duplex_continuous_audio.sh`;它内部继续复用同一套
|
| 6 |
+
`cache_omni_hidden_states.py` 在线等价编码逻辑。
|
| 7 |
+
|
| 8 |
+
## 正式全双工连续背景数据
|
| 9 |
+
|
| 10 |
+
```bash
|
| 11 |
+
bash prepare_full_duplex_continuous_audio.sh
|
| 12 |
+
```
|
| 13 |
+
|
| 14 |
+
默认使用可见的四张 GPU、每卡四个独立 batch-1 Qwen 进程,结果写入:
|
| 15 |
+
|
| 16 |
+
```text
|
| 17 |
+
/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/videogen/hfd_audio_command_continuous_office_v1
|
| 18 |
+
```
|
| 19 |
+
|
| 20 |
+
每条 TTS source 会固定绑定一条不循环、不拼接的连续办公室背景时间线:
|
| 21 |
+
|
| 22 |
+
- idle 读取语音开始前的相邻纯背景窗口;
|
| 23 |
+
- listen 先在波形层叠加同一段背景与人声,再单窗编码;
|
| 24 |
+
- motion 从语音结束后的下一个相邻纯背景窗口继续。
|
| 25 |
+
|
| 26 |
+
相邻 0.6 秒窗口以 0.4 秒滑动,边界共享的 0.2 秒 PCM 必须逐样本相同。
|
| 27 |
+
脚本会生成 41,694 条 mixed WAV、388,374 个 mixed hidden 窗口,以及
|
| 28 |
+
10 条背景 session 的 8,960 个可复用纯背景 hidden 窗口。全部波形、索引、
|
| 29 |
+
encoder contract 和独立在线 Qwen 复算通过后,才会写
|
| 30 |
+
`reports/production_complete.json`;正式训练和 semantics-v3 推理都要求该
|
| 31 |
+
标记及 assignment SHA-256 一致。
|
| 32 |
+
|
| 33 |
+
## 怎么用
|
| 34 |
+
|
| 35 |
+
1. 只修改脚本开头“只需要改这一段”:
|
| 36 |
+
- `OUTPUT_KIND=embeds`:保留 VLM,可训练/解冻 VLM。
|
| 37 |
+
- `OUTPUT_KIND=hidden`:冻结 VLM,生成可直接训练的 hidden-state 数据。
|
| 38 |
+
- 默认 `HIDDEN_LAYOUT=source` 只生成 HFD audio-command;旧多任务
|
| 39 |
+
大分片流程可显式改成 `HIDDEN_LAYOUT=oss`。
|
| 40 |
+
- 再设置 `OUTPUT_ROOT`、`AUDIO_WAV_ROOT`、`SELECTED_TASKS`、`SELECTED_SOURCES` 和 GPU。
|
| 41 |
+
2. 正式生成必须保持 `LIMIT=0`。
|
| 42 |
+
3. 运行:
|
| 43 |
+
|
| 44 |
+
```bash
|
| 45 |
+
bash prepare_omni_training_data.sh
|
| 46 |
+
```
|
| 47 |
+
|
| 48 |
+
脚本不接收位置参数,所有常用选项都集中在文件顶部。
|
| 49 |
+
|
| 50 |
+
## 任务和数据源
|
| 51 |
+
|
| 52 |
+
| 任务 | 可用数据源 |
|
| 53 |
+
|---|---|
|
| 54 |
+
| `text` | `seed_text` |
|
| 55 |
+
| `audio_text` / `audio_only` | `psi0_dance_audio`、`2264_with_audio` |
|
| 56 |
+
| `audio_command_text` / `audio_command_only` | `qwen3_tts_seed_audio_command` |
|
| 57 |
+
| `video_clip_text` / `video_clip_only` | 三个 `*_clip50` 视频源;`distilled_video_0226_clip50` 暂只用于 embeds |
|
| 58 |
+
|
| 59 |
+
完整数据源名称已经列在脚本顶部注释中。
|
| 60 |
+
|
| 61 |
+
## 音频输入
|
| 62 |
+
|
| 63 |
+
- 统一接口现在只读 WAV,不再把 NPY 送进 processor/VLM。
|
| 64 |
+
- `audio_command` 直接读取 Qwen3-TTS 的原始 `seed_audio/*/*.wav`。
|
| 65 |
+
- dance 和 2264 原始数据只有 MP4 音轨,没有独立 WAV。为了与历史训练逐采样一致,脚本会把旧 NPY 中已经确定的 16 kHz 单声道 float32 采样无损封装成 WAV;dance manifest 中回退到 MP4 的 25 条经检查都没有音轨,和旧 exporter 一样明确跳过。结果保存到 `AUDIO_WAV_ROOT`,每个 WAV 写完都会解码回读并做逐元素校验。
|
| 66 |
+
- `AUDIO_WINDOW_SECONDS` 控制 `audio_text` 的窗口长度,`AUDIO_SAMPLE_STEP_FRAMES` 控制训练窗口按多少个 50 FPS 动作帧滑动。默认是 0.6 秒、逐帧滑动。
|
| 67 |
+
- `audio_command` 使用独立配置:默认只保留完整的 0.6 秒窗口,并按 20 个动作帧(0.4 秒)滑动;最后不足 0.6 秒的尾段不补零。当前 41,760 条 WAV 会生成 304,986 个窗口。把 `AUDIO_COMMAND_FULL_AUDIO` 设为 `1` 才会恢复旧版整段语音行为。
|
| 68 |
+
|
| 69 |
+
旧版 WAV/媒体转 NPY 的规则已经核对过:ffmpeg 输出 16 kHz、单声道、`f32le` 后由 `numpy.save` 保存。新 WAV 解码沿用完全相同的 ffmpeg 参数。
|
| 70 |
+
|
| 71 |
+
## 输出是什么
|
| 72 |
+
|
| 73 |
+
`embeds` 输出:
|
| 74 |
+
|
| 75 |
+
- `<OUTPUT_ROOT>/<视频源>/metadata.jsonl`
|
| 76 |
+
- `<OUTPUT_ROOT>/<视频源>/embeds/*.npz`
|
| 77 |
+
- `<OUTPUT_ROOT>/training_support/source_split_v1/`
|
| 78 |
+
|
| 79 |
+
默认 `hidden/source` 输出:
|
| 80 |
+
|
| 81 |
+
- `<OUTPUT_ROOT>/audio/qwen3_tts_seed_audio_command/audio_command_text/sources/seed_audio/*/*.safetensors`
|
| 82 |
+
- 同目录的少量 `index.ws*.rank*.sqlite` source 索引
|
| 83 |
+
- `<OUTPUT_ROOT>/_reports/encoder_contract.json`
|
| 84 |
+
- `<OUTPUT_ROOT>/_reports/online_hidden_verify.json`
|
| 85 |
+
- `<OUTPUT_ROOT>/_reports/prepare_complete.json`
|
| 86 |
+
|
| 87 |
+
每个有效 WAV 对应一个 safetensors,文件内部按时间顺序保存该 WAV
|
| 88 |
+
全部 0.6 秒 hidden 窗口、offset 和 frame id。41,760 条原始 WAV 中有
|
| 89 |
+
41,694 条至少包含一个完整窗口,共 304,986 个窗口。生成过程直接写最终
|
| 90 |
+
source 文件,不创建 92 GiB staging 副本,不做二次打乱/merge,也不写
|
| 91 |
+
30 万行逐窗口 JSONL/SQLite。每个窗口都单独以 batch 1 送入 Qwen2.5-Omni,
|
| 92 |
+
保存 thinker 最后一层 bf16 hidden;这样与真实流式推理的单窗口编码逐元素一致。
|
| 93 |
+
|
| 94 |
+
`HIDDEN_LAYOUT=oss` 保留旧行为:同一任务内确定性打乱,并合并成最多
|
| 95 |
+
4096 条或 4.5 GiB 的 shard,同时生成旧训练代码兼容索引。
|
| 96 |
+
|
| 97 |
+
## 旧版独立 speech-only HFD hidden
|
| 98 |
+
|
| 99 |
+
通用入口��认固定选择 `audio_command_text` 和 Qwen3-TTS 数据源,
|
| 100 |
+
使用 0.6 秒窗口、20 帧/0.4 秒步长,并写入与旧大分片数据隔离的目录:
|
| 101 |
+
|
| 102 |
+
```text
|
| 103 |
+
/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/videogen/hfd_audio_command_source
|
| 104 |
+
```
|
| 105 |
+
|
| 106 |
+
默认使用 PyTorch 实际识别到的全部可见 GPU、每卡八个独立 batch-1 导出进程;
|
| 107 |
+
增加吞吐靠多个独立模型副本,不会把多个窗口放进一次前向而改变数值。
|
| 108 |
+
`OVERWRITE=0` 时会验证并复用完整 source 文件。`LIMIT` 在 source 模式表示
|
| 109 |
+
完整 WAV 数量,只用于测试;有限测试只写
|
| 110 |
+
`prepare_debug_limit_*.json`,不会伪装成正式完成数据。旧 semantics-v2
|
| 111 |
+
入口曾固定读取这个 source 目录;完成前还会抽取完整 WAV,通过真实在线 batch-1
|
| 112 |
+
Qwen 路径逐窗口
|
| 113 |
+
复算,只有 `mismatched_windows=0` 且 `max_abs_error=0` 才写完成标记。训练和推理
|
| 114 |
+
启动时都会校验模型指纹、固定提示词、dtype、attention 实现和在线复核报告;任何
|
| 115 |
+
旧缓存或混入的旧 source 文件都会直接报错。只有显式设置
|
| 116 |
+
`HFD_AUDIO_COMMAND_ROOT` 才能覆盖数据目录。
|
| 117 |
+
|
| 118 |
+
这份 speech-only 缓存只用于旧 semantics-v2 复现或重新迁移 source 清单;当前
|
| 119 |
+
semantics-v3 正式训练读取上一节的连续背景 production 数据,不再读取该目录。
|
| 120 |
+
|
| 121 |
+
## 安全规则
|
| 122 |
+
|
| 123 |
+
- 默认 `OVERWRITE=0`,已有有效结果会复用。
|
| 124 |
+
- source 模式没有大 staging;只有单个正在保存的原子临时文件。
|
| 125 |
+
- OSS 模式只有全部生成、索引和抽样校验成功后,才会删除中间 staging。
|
| 126 |
+
- 最终完成标记是 `_reports/prepare_complete.json`;它只会在在线逐元素复核成功后
|
| 127 |
+
写入,没有它就不要上传或训练。
|
| 128 |
+
- 旧入口保存在 `backup/legacy_data_preprocess/`,只用于追溯和回退。
|
Psi0-训练阶段-state-action总览.md
ADDED
|
@@ -0,0 +1,216 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
---
|
| 2 |
+
tags:
|
| 3 |
+
- psi0
|
| 4 |
+
- training
|
| 5 |
+
- state-action
|
| 6 |
+
---
|
| 7 |
+
|
| 8 |
+
# Psi0 训练阶段 state/action 总览
|
| 9 |
+
|
| 10 |
+
> [!summary]
|
| 11 |
+
> 当前开源代码里,`pretrain`、`posttrain`、`finetune-real`、`finetune-simple` 都不是“显式全身关节输入 + 显式全身关节输出”。
|
| 12 |
+
> 更接近的实际接口是:
|
| 13 |
+
> - 上肢:手 + 手臂
|
| 14 |
+
> - 躯干:`rpy` + `height`
|
| 15 |
+
> - 移动相关:`torso_vx` / `torso_vy` / `torso_vyaw` / `target_yaw`
|
| 16 |
+
> - 下肢:更多像交给低层 controller,而不是由这层模型直接输出腿关节轨迹
|
| 17 |
+
|
| 18 |
+
## 1. 一页结论
|
| 19 |
+
|
| 20 |
+
| 阶段 | 模型输入 `state` | 模型输出 `action` | 是否显式包含腿关节输出 | 依据 |
|
| 21 |
+
| --- | --- | --- | --- | --- |
|
| 22 |
+
| `pretrain` `EgoDex` | `48` 维手部/腕部表示 | `48` 维手部/腕部表示 | 否 | `src/psi/data/egodex/egodex_dataset.py:308-330, 519-523` |
|
| 23 |
+
| `pretrain` `HE` | `hand_joints + arm_joints` | 左右腕位姿增量 + 双手指尖轨迹 | 否 | `src/psi/config/transform.py:112-212, 221-230` |
|
| 24 |
+
| `posttrain` `HE` | `hand_joints + arm_joints` | `action.joint_angles`,但当前接口仍是上肢维度 | 否 | `src/psi/data/humanoid/he_raw_dataset.py:206-229`;`src/psi/config/transform.py:718-755` |
|
| 25 |
+
| `finetune-real` | `hand + arm + torso_rpy + torso_height` | `hand + arm + rpy + height + vx + vy + vyaw + target_yaw` | 否 | `scripts/data/raw_to_lerobot.py:203-212, 239-258` |
|
| 26 |
+
| `finetune-simple` | `32` 维 psi-style `states`,本质是 `hand + arm + rpy + height` | `36` 维动作,核心仍是上肢 + 躯干/移动命令 | 否 | `src/gr00t/gr00t/deploy/gr00t_serve_simple.py:193-219, 133-166`;`src/psi/config/transform.py:376-393` |
|
| 27 |
+
|
| 28 |
+
## 2. 各阶段拆开看
|
| 29 |
+
|
| 30 |
+
### 2.1 `pretrain` on `EgoDex`
|
| 31 |
+
|
| 32 |
+
- `states` 是当前动作表示,不是全身关节。
|
| 33 |
+
- `actions` 是未来动作 chunk,也不是全身关节。
|
| 34 |
+
- 代码里直接写的是 `48d` action representation。
|
| 35 |
+
|
| 36 |
+
维度含义:
|
| 37 |
+
- 左腕
|
| 38 |
+
- 左手指尖
|
| 39 |
+
- 右腕
|
| 40 |
+
- 右手指尖
|
| 41 |
+
|
| 42 |
+
依据:
|
| 43 |
+
- `src/psi/data/egodex/egodex_dataset.py:308-330`
|
| 44 |
+
- `src/psi/data/egodex/egodex_dataset.py:519-523`
|
| 45 |
+
|
| 46 |
+
结论:
|
| 47 |
+
- 这是手部/腕部操作表示,不是全身关节表示。
|
| 48 |
+
|
| 49 |
+
### 2.2 `pretrain` on `HE`
|
| 50 |
+
|
| 51 |
+
- `state` 由 `observation.hand_joints` 和 `observation.arm_joints` 拼起来。
|
| 52 |
+
- `action` 由左右腕位姿和双手指尖轨迹拼起来。
|
| 53 |
+
- 没有把腿关节放进这条训练链路。
|
| 54 |
+
|
| 55 |
+
依据:
|
| 56 |
+
- `src/psi/config/transform.py:221-230`
|
| 57 |
+
- `src/psi/config/transform.py:112-212`
|
| 58 |
+
|
| 59 |
+
结论:
|
| 60 |
+
- 这条线仍然是上肢操作表示,不是全身关节表示。
|
| 61 |
+
|
| 62 |
+
### 2.3 `posttrain` on `HE`
|
| 63 |
+
|
| 64 |
+
- 原始 HE dataset loader 只读:
|
| 65 |
+
- `hand_state`
|
| 66 |
+
- `arm_state`
|
| 67 |
+
- 没有在这里读 `leg_state` 进入训练样本。
|
| 68 |
+
- `HEPosttrainRepackTransform` 里:
|
| 69 |
+
- `states = hand_joints + arm_joints`
|
| 70 |
+
- `actions = action.joint_angles`
|
| 71 |
+
- 但当前接口和脚本配置仍然是上肢维度,不是完整腿部关节轨迹接口。
|
| 72 |
+
|
| 73 |
+
依据:
|
| 74 |
+
- `src/psi/data/humanoid/he_raw_dataset.py:206-229`
|
| 75 |
+
- `src/psi/config/transform.py:718-755`
|
| 76 |
+
|
| 77 |
+
结论:
|
| 78 |
+
- `posttrain` 也不是显式全身关节训练。
|
| 79 |
+
|
| 80 |
+
### 2.4 `finetune-real`
|
| 81 |
+
|
| 82 |
+
- 数据转换脚本确实读了 `leg_state`,也把 `sol_q` 拆出了 `leg_joints`。
|
| 83 |
+
- 但最后真正写进 `states` / `action` 的内容里,没有把腿关节放进去。
|
| 84 |
+
|
| 85 |
+
`states` 实际组成:
|
| 86 |
+
- `hand_joints`
|
| 87 |
+
- `arm_joints`
|
| 88 |
+
- `torso_rpy`
|
| 89 |
+
- `torso_height`
|
| 90 |
+
|
| 91 |
+
`action` 实际组成:
|
| 92 |
+
- `hand_joints`
|
| 93 |
+
- `arm_joints`
|
| 94 |
+
- `rpy`
|
| 95 |
+
- `height`
|
| 96 |
+
- `torso_vx`
|
| 97 |
+
- `torso_vy`
|
| 98 |
+
- `torso_vyaw`
|
| 99 |
+
- `target_yaw`
|
| 100 |
+
|
| 101 |
+
依据:
|
| 102 |
+
- `scripts/data/raw_to_lerobot.py:203-212`
|
| 103 |
+
- `scripts/data/raw_to_lerobot.py:239-258`
|
| 104 |
+
|
| 105 |
+
训练器对 `36` 维动作的切分也和上面一致:
|
| 106 |
+
- `0:14` 手
|
| 107 |
+
- `14:28` 臂
|
| 108 |
+
- `28:31` `rpy`
|
| 109 |
+
- `31:32` `height`
|
| 110 |
+
- `32:33` `vx`
|
| 111 |
+
- `33:34` `vy`
|
| 112 |
+
- `34:35` `vyaw`
|
| 113 |
+
- `35:36` `target_yaw`
|
| 114 |
+
|
| 115 |
+
依据:
|
| 116 |
+
- `src/psi/trainers/finetune.py:492-520`
|
| 117 |
+
|
| 118 |
+
脚本配置也写死了 `36` 维:
|
| 119 |
+
- `scripts/train/psi0/finetune-real-psi0.sh:50-72`
|
| 120 |
+
|
| 121 |
+
补充:
|
| 122 |
+
- 可视化脚本也把 real 数据的腿关节直接置零,不从预测动作里取。
|
| 123 |
+
|
| 124 |
+
依据:
|
| 125 |
+
- `scripts/viz/viz_episode_real.py:73-75`
|
| 126 |
+
- `scripts/viz/viz_episode_real.py:90-94`
|
| 127 |
+
|
| 128 |
+
结论:
|
| 129 |
+
- `finetune-real` 不是“全身关节预测”。
|
| 130 |
+
- 更准确地说,是“上肢关节 + 躯干/移动命令预测”。
|
| 131 |
+
|
| 132 |
+
### 2.5 `finetune-simple`
|
| 133 |
+
|
| 134 |
+
- `simple` 原始观测里有腿关节,这点是有的。
|
| 135 |
+
- 但给 psi 模型用的 `states` 不是原始全身 `proprio`,而是压缩过的 psi-style 状态。
|
| 136 |
+
|
| 137 |
+
`states` 的 `32` 维组成:
|
| 138 |
+
- 左手 `7`
|
| 139 |
+
- 右手 `7`
|
| 140 |
+
- 左臂 `7`
|
| 141 |
+
- 右臂 `7`
|
| 142 |
+
- `rpy` `3`
|
| 143 |
+
- `height` `1`
|
| 144 |
+
|
| 145 |
+
依据:
|
| 146 |
+
- `src/gr00t/gr00t/deploy/gr00t_serve_simple.py:193-219`
|
| 147 |
+
|
| 148 |
+
`action` 在 SIMPLE 服务侧被拆成:
|
| 149 |
+
- `left_hand`
|
| 150 |
+
- `right_hand`
|
| 151 |
+
- `left_arm`
|
| 152 |
+
- `right_arm`
|
| 153 |
+
- `rpy`
|
| 154 |
+
- `height`
|
| 155 |
+
- `torso_vx`
|
| 156 |
+
- `torso_vy`
|
| 157 |
+
- `torso_vyaw`
|
| 158 |
+
- `target_yaw`
|
| 159 |
+
|
| 160 |
+
依据:
|
| 161 |
+
- `src/gr00t/gr00t/deploy/gr00t_serve_simple.py:133-166`
|
| 162 |
+
|
| 163 |
+
psi 的 `SimpleRepackTransform` 也只是把 `states` / `action` 取进来,再 pad 到训练���度:
|
| 164 |
+
- `src/psi/config/transform.py:376-393`
|
| 165 |
+
|
| 166 |
+
训练脚本同样固定成 `36` 维:
|
| 167 |
+
- `scripts/train/psi0/finetune-simple-psi0.sh:50-72`
|
| 168 |
+
|
| 169 |
+
可视化脚本里:
|
| 170 |
+
- `proprio` 会显示腿关节
|
| 171 |
+
- 但 `action` 里的腿是直接用当前观测腿关节,而不是用模型预测结果
|
| 172 |
+
|
| 173 |
+
依据:
|
| 174 |
+
- `scripts/viz/viz_episode_simple.py:83-85`
|
| 175 |
+
- `scripts/viz/viz_episode_simple.py:93-96`
|
| 176 |
+
|
| 177 |
+
结论:
|
| 178 |
+
- `finetune-simple` 这条线也不是“模型直接输出腿关节轨迹”。
|
| 179 |
+
|
| 180 |
+
## 3. 下肢到底在哪里
|
| 181 |
+
|
| 182 |
+
> [!note]
|
| 183 |
+
> README 的说法和代码里的实际接口,要分开理解。
|
| 184 |
+
|
| 185 |
+
README 里写的是:
|
| 186 |
+
- `System-1` 预测 future whole-body action chunks
|
| 187 |
+
- `System-0` 执行 predicted lower-body action commands
|
| 188 |
+
|
| 189 |
+
依据:
|
| 190 |
+
- `README.md:26`
|
| 191 |
+
|
| 192 |
+
结合实际代码,更合理的理解是:
|
| 193 |
+
- `System-1` 负责高层动作生成
|
| 194 |
+
- 当前开源训练接口里,显式暴露给模型训练的主要是上肢和躯干/移动命令
|
| 195 |
+
- 下肢稳定控制更多由 `System-0` 的 tracking controller 负责
|
| 196 |
+
|
| 197 |
+
所以如果问题是:
|
| 198 |
+
|
| 199 |
+
**“这份代码里有没有直接训练/输出全身每个关节?”**
|
| 200 |
+
|
| 201 |
+
答案是:
|
| 202 |
+
- 没看到。
|
| 203 |
+
|
| 204 |
+
**“有没有和移动、下肢相关的信息?”**
|
| 205 |
+
|
| 206 |
+
答案是:
|
| 207 |
+
- 有,但更像高层 locomotion command,不是显式腿关节序列。
|
| 208 |
+
|
| 209 |
+
## 4. 最短版记忆卡
|
| 210 |
+
|
| 211 |
+
- `pretrain(EgoDex)`:手/腕 `48D`
|
| 212 |
+
- `pretrain(HE)`:`hand + arm`,动作是腕位姿 + 指尖轨迹
|
| 213 |
+
- `posttrain(HE)`:`hand + arm`
|
| 214 |
+
- `finetune-real`:`hand + arm + rpy + height + vx/vy/vyaw/target_yaw`
|
| 215 |
+
- `finetune-simple`:原始观测有腿,但 psi 模型接口仍不是显式腿关节输出
|
| 216 |
+
- 结论:当前开源接口不是“全身关节端到端直接预测”
|
README.md
ADDED
|
@@ -0,0 +1,532 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
<h1 align="center">Ψ₀: An Open Foundation Model <br/> Towards Universal Humanoid Loco-Manipulation
|
| 2 |
+
</h1>
|
| 3 |
+
|
| 4 |
+
<p align="center">
|
| 5 |
+
<img src="assets/media/teaser.jpg" alt="Psi0 teaser image" />
|
| 6 |
+
</p>
|
| 7 |
+
|
| 8 |
+
<div align="center">
|
| 9 |
+
|
| 10 |
+
[](https://arxiv.org/abs/2603.12263)
|
| 11 |
+
[](https://psi-lab.ai/Psi0)
|
| 12 |
+
[](https://huggingface.co/USC-PSI-Lab/psi-model)
|
| 13 |
+
[](https://huggingface.co/datasets/USC-PSI-Lab/psi-data)
|
| 14 |
+
[](./LICENSE)
|
| 15 |
+
|
| 16 |
+
</div>
|
| 17 |
+
|
| 18 |
+
Contributors: [Songlin Wei](https://songlin.github.io/), [Hongyi Jing](https://hongyijing.me/), [Boqian Li](https://boqian-li.github.io/), [Zhenyu Zhao](https://zhenyuzhao.com/), [Jiageng Mao](https://pointscoder.github.io/), [Zhenhao Ni](https://nizhenhao-3.github.io/) , [Sicheng He](https://hesicheng.net/), [Jie Liu](https://jie0530.github.io/), [Xiawei Liu](https://www.xiaweiliu.com/), Kaidi Kang, Sheng Zang,[Weiduo Yuan](https://weiduoyuan.com/), [Marco Pavone](https://profiles.stanford.edu/marco-pavone), Di Huang, [Yue Wang](https://yuewang.xyz/)
|
| 19 |
+
|
| 20 |
+
-------
|
| 21 |
+
|
| 22 |
+
$\Psi_0$ is an open vision-language-action (VLA) model for dexterous humanoid loco-manipulation. Our model first learns task semantics and visual representation from large-scale human egocentic videos, and then is post-trained on a smaller amount of real-world teleoperated robot data, to learn general dynamics of the embodiment.
|
| 23 |
+
|
| 24 |
+
Our foundation model is capable of acquiring new long-horizontal dexterous loco-manipulation skill by fine-tuning using as few as 80 trajectories. ***Our key finding is that scaling the right data in the right way.***
|
| 25 |
+
|
| 26 |
+
At the top, the $\Psi_0$ model consists of two end-to-end trained components: a vision–language backbone (System-2) and a multimodal diffusion transformer (System-1) action expert. The backbone is based on Qwen’s Qwen3-VL-2B-Instruct, which extracts vision–language features from observations and instructions. These features condition a flow-based multimodal diffusion transformer inspired by Stable Diffusion 3. The action expert (≈500M parameters) predicts future whole-body action chunks, enabling efficient fusion of visual, linguistic, and action representations. At the lowest level (System-0), an RL-based tracking controller executes the predicted lower-body action commands, ensuring stable and precise physical control.
|
| 27 |
+
|
| 28 |
+
<p align="center">
|
| 29 |
+
<img src="assets/media/arch.png" alt="Psi0 model" />
|
| 30 |
+
</p>
|
| 31 |
+
|
| 32 |
+
## Table of Contents
|
| 33 |
+
<!-- - [Installation](#-environment-setup) -->
|
| 34 |
+
<!-- - [Pre- & Post- Training](#-) -->
|
| 35 |
+
<!-- - [Data Pre-Processing](#-) -->
|
| 36 |
+
- [Finetune Ψ₀ on Unitree G1 Humanoid Robot](#finetune-psi0)
|
| 37 |
+
- [Installation](#installation)
|
| 38 |
+
- [Data Collection](#data-collection)
|
| 39 |
+
- [Fine-Tuning](#training-real)
|
| 40 |
+
- [Open-Loop Evaluation](#open-loop-evaluation)
|
| 41 |
+
- [Deployment](#deployment)
|
| 42 |
+
- [Baselines](#baselines)
|
| 43 |
+
- [GR00T N1.6](#groot-n16)
|
| 44 |
+
- [OpenPi π0.5](#openpi-05)
|
| 45 |
+
- [InternVLA-M1](#internvla-m1)
|
| 46 |
+
- [H-RDT](#-)
|
| 47 |
+
- [EgoVLA](#-)
|
| 48 |
+
- [Diffusion Policy](#-)
|
| 49 |
+
- [ACT](#-)
|
| 50 |
+
- [Simulation](#simulation)
|
| 51 |
+
- [Install SIMPLE](#install-simple)
|
| 52 |
+
- [Data Generation](#data-generation)
|
| 53 |
+
- [Fine-Tuning](#training-sim)
|
| 54 |
+
- [Evaluation in SIMPLE](#evaluation-in-simple)
|
| 55 |
+
- [Reproduce Ψ₀: Pre-Training and Post-Training](#pre-post-train)
|
| 56 |
+
- [Checkpoints](#checkpoints)
|
| 57 |
+
- [Troubleshootings](#troubleshootings)
|
| 58 |
+
- [Citation](#️-citation)
|
| 59 |
+
|
| 60 |
+
<a id="finetune-psi0"></a>
|
| 61 |
+
## Finetune Ψ₀ on Unitree G1 Humanoid Robot
|
| 62 |
+
|
| 63 |
+
### Installation
|
| 64 |
+
|
| 65 |
+
Clone the project and change directory to the project root:
|
| 66 |
+
```bash
|
| 67 |
+
git clone git@github.com:physical-superintelligence-lab/Psi0.git
|
| 68 |
+
cd Psi0
|
| 69 |
+
```
|
| 70 |
+
We use [uv](https://docs.astral.sh/uv/getting-started/installation/) to manage Python dependencies. Install `uv` if not already installed:
|
| 71 |
+
|
| 72 |
+
```bash
|
| 73 |
+
curl -LsSf https://astral.sh/uv/install.sh | sh
|
| 74 |
+
```
|
| 75 |
+
|
| 76 |
+
Set up the $\Psi_0$ environment:
|
| 77 |
+
|
| 78 |
+
> ℹ️ We manage the $\Psi_0$ environment and all the baselines through `uv` and they all share the same `src/` code. See [Environment Management](baselines/README.md) for more details.
|
| 79 |
+
|
| 80 |
+
```
|
| 81 |
+
uv venv .venv-psi --python 3.10
|
| 82 |
+
source .venv-psi/bin/activate
|
| 83 |
+
GIT_LFS_SKIP_SMUDGE=1 uv sync --all-groups --index-strategy unsafe-best-match --active
|
| 84 |
+
uv pip install flash_attn==2.7.4.post1 --no-build-isolation
|
| 85 |
+
```
|
| 86 |
+
|
| 87 |
+
Test installation, a version number should be displayed.
|
| 88 |
+
```bash
|
| 89 |
+
python -c "import psi;print(psi.__version__)"
|
| 90 |
+
```
|
| 91 |
+
|
| 92 |
+
Verify the shared `lerobot` stack is importable.
|
| 93 |
+
```bash
|
| 94 |
+
python -c "from psi.data.lerobot.compat import LEROBOT_LAYOUT; print(LEROBOT_LAYOUT)"
|
| 95 |
+
```
|
| 96 |
+
|
| 97 |
+
### Data Collection
|
| 98 |
+
> 📂 We open-sourced all the 9 real-world tasks. You can directly download the data and jump to the [Fine-Tuning](#training-real).
|
| 99 |
+
|
| 100 |
+
> 🔥 We first release our internal test data collection pipeline which uses Apple Vision Pro to teleoperate Unitree G1 humanoid Robot with two Dex3-1 hands.
|
| 101 |
+
|
| 102 |
+
See the detailed teleoperation guide here:
|
| 103 |
+
[Real-World Teleoperation Guide](real/README.md)
|
| 104 |
+
|
| 105 |
+
|
| 106 |
+
#### Pre-Processing: Convert Raw Data to LeRobot Format
|
| 107 |
+
|
| 108 |
+
```
|
| 109 |
+
export task=Hug_box_and_move
|
| 110 |
+
|
| 111 |
+
hf download USC-PSI-Lab/psi-data \
|
| 112 |
+
g1_real_raw/$task.zip \
|
| 113 |
+
--local-dir=$PSI_HOME/data/real_teleop_g1 \
|
| 114 |
+
--repo-type=dataset
|
| 115 |
+
|
| 116 |
+
unzip $PSI_HOME/data/real_teleop_g1/g1_real_raw/$task.zip -d $PSI_HOME/data/real_teleop_g1/g1_real_raw/$task
|
| 117 |
+
```
|
| 118 |
+
You should observe similar folder structure:
|
| 119 |
+
|
| 120 |
+
```
|
| 121 |
+
g1_real_raw
|
| 122 |
+
└── Hug_box_and_move
|
| 123 |
+
├── episode_0
|
| 124 |
+
│ ├── color
|
| 125 |
+
│ │ ├── frame_000000.jpg
|
| 126 |
+
│ │ └── ...
|
| 127 |
+
│ └── data.json
|
| 128 |
+
└── ...
|
| 129 |
+
```
|
| 130 |
+
|
| 131 |
+
Edit the task description file with the following format, eg.,
|
| 132 |
+
```
|
| 133 |
+
vim scripts/data/task_description_dict.json
|
| 134 |
+
```
|
| 135 |
+
```
|
| 136 |
+
{
|
| 137 |
+
"Hug_box_and_move": "Hug box and move."
|
| 138 |
+
}
|
| 139 |
+
```
|
| 140 |
+
|
| 141 |
+
Run conversion script
|
| 142 |
+
```
|
| 143 |
+
python scripts/data/raw_to_lerobot.py \
|
| 144 |
+
--data-root=/hfm/data/real_teleop_g1/g1_real_raw \
|
| 145 |
+
--work-dir=/hfm/data/real \
|
| 146 |
+
--repo-id=psi0-real-g1 \
|
| 147 |
+
--robot-type=g1 \
|
| 148 |
+
--task=$task
|
| 149 |
+
```
|
| 150 |
+
|
| 151 |
+
Calculate stats
|
| 152 |
+
```
|
| 153 |
+
python scripts/data/calc_modality_stats.py \
|
| 154 |
+
--work-dir=$PSI_HOME/data/real \
|
| 155 |
+
--task=$task
|
| 156 |
+
```
|
| 157 |
+
|
| 158 |
+
Create **$\Psi_0$** format stats (simply a copy for now)
|
| 159 |
+
```
|
| 160 |
+
cp $PSI_HOME/data/real/$task/meta/stats.json $PSI_HOME/data/real/$task/meta/stats_psi0.json
|
| 161 |
+
```
|
| 162 |
+
|
| 163 |
+
Now it's ready to finetune $\Psi_0$.
|
| 164 |
+
|
| 165 |
+
> ✈️ If training env is already configured, directly launch training via `scripts/train/psi0/finetune-real-psi0.sh $task`
|
| 166 |
+
|
| 167 |
+
|
| 168 |
+
<a id="training-real"></a>
|
| 169 |
+
### Fine-Tuning
|
| 170 |
+
|
| 171 |
+
> ✔️ Suppose the data is already collected and processed. Now we can proceed to fine-tune the $\Psi_0$ model.
|
| 172 |
+
|
| 173 |
+
> There is a [known issue](https://github.com/physical-superintelligence-lab/Psi0/issues/3) of loading our real data, apply this fix first `python scripts/data/patch_lerobot_meta.py $PSI_HOME/data/real/$task`
|
| 174 |
+
|
| 175 |
+
> 📝 Here we illustrate by using the pre-collected data from [Huggingface psi-data](https://huggingface.co/datasets/USC-PSI-Lab/psi-data/tree/main/real).
|
| 176 |
+
|
| 177 |
+
Set up the environment variables following `.env.sample`. The environment variables will be loaded by the `dotenv.load_dotenv()` in python.
|
| 178 |
+
```
|
| 179 |
+
cp .env.sample .env
|
| 180 |
+
# and edit the following env variables
|
| 181 |
+
# HF_TOKEN=<YOUR HF READ TOKEN>
|
| 182 |
+
# WANDB_API_KEY=<API KEY for wandb logging>
|
| 183 |
+
# WANDB_ENTITY=<wandb entity>
|
| 184 |
+
# PSI_HOME=<Path where PSI cache/checkpoint/data are located by convention>
|
| 185 |
+
|
| 186 |
+
source .env
|
| 187 |
+
echo $PSI_HOME
|
| 188 |
+
```
|
| 189 |
+
|
| 190 |
+
Download the collected real-world data and extract it:
|
| 191 |
+
```
|
| 192 |
+
export task=Pick_bottle_and_turn_and_pour_into_cup
|
| 193 |
+
|
| 194 |
+
hf download USC-PSI-Lab/psi-data \
|
| 195 |
+
real/$task.zip \
|
| 196 |
+
--local-dir=$PSI_HOME/data \
|
| 197 |
+
--repo-type=dataset
|
| 198 |
+
|
| 199 |
+
unzip $PSI_HOME/data/real/$task.zip -d $PSI_HOME/data/real
|
| 200 |
+
```
|
| 201 |
+
> 👀 If you want to visualize the episode please refer to the [Data Visualization](examples/visualize.md) in the examples.
|
| 202 |
+
|
| 203 |
+
Launch the training script:
|
| 204 |
+
```
|
| 205 |
+
scripts/train/psi0/finetune-real-psi0.sh $task
|
| 206 |
+
```
|
| 207 |
+
|
| 208 |
+
> 🖥️ You can always change the GPUs, e.g., `CUDA_VISIBLE_DEVICES=0,1,2,3 scripts/train/...`.
|
| 209 |
+
|
| 210 |
+
> ⚠️ Please try to maintain a reasonable global batch size = device batch size x number of GPUs x gradient accumulation step. We use global batch size 128 throughout all the real-world and simulation experiments.
|
| 211 |
+
|
| 212 |
+
|
| 213 |
+
### Open-Loop Evaluation
|
| 214 |
+
> Follow the steps in `examples/simple/openloop_eval.ipynb`
|
| 215 |
+
|
| 216 |
+
Load the training dataset, and run model inference to see how model fits the training data.
|
| 217 |
+
|
| 218 |
+
### Deployment
|
| 219 |
+
|
| 220 |
+
#### Serve $\Psi_0$ (RTC mode)
|
| 221 |
+
|
| 222 |
+
```bash
|
| 223 |
+
bash ./scripts/deploy/serve_psi0-rtc.sh
|
| 224 |
+
```
|
| 225 |
+
|
| 226 |
+
#### Start $\Psi_0$ Client (RTC mode)
|
| 227 |
+
|
| 228 |
+
```bash
|
| 229 |
+
bash ./real/scripts/deploy_psi0-rtc.sh
|
| 230 |
+
```
|
| 231 |
+
|
| 232 |
+
For detailed real-world deployment environment setup, please also refer to the dedicated documentation:
|
| 233 |
+
|
| 234 |
+
[Real-World Teleoperation Guide](real/README.md)
|
| 235 |
+
|
| 236 |
+
|
| 237 |
+
## Baselines
|
| 238 |
+
|
| 239 |
+
<a id="groot-n16"></a>
|
| 240 |
+
|
| 241 |
+
### GR00T
|
| 242 |
+
Install the env
|
| 243 |
+
```bash
|
| 244 |
+
cd src/gr00t; uv sync
|
| 245 |
+
```
|
| 246 |
+
1. training
|
| 247 |
+
```bash
|
| 248 |
+
cd src/gr00t
|
| 249 |
+
./scripts/train_gr00t.sh --dataset-path /your/lerobot/dataset
|
| 250 |
+
```
|
| 251 |
+
2. serving a checkpoint
|
| 252 |
+
```bash
|
| 253 |
+
cd src/gr00t
|
| 254 |
+
./scripts/deploy_gr00t.sh
|
| 255 |
+
```
|
| 256 |
+
|
| 257 |
+
3. openloop eval on trained checkpoint using gt
|
| 258 |
+
```bash
|
| 259 |
+
cd src/gr00t
|
| 260 |
+
./scripts/openloop_eval.sh
|
| 261 |
+
```
|
| 262 |
+
|
| 263 |
+
<a id="openpi-05"></a>
|
| 264 |
+
|
| 265 |
+
### OpenPI $\pi_{0.5}$
|
| 266 |
+
|
| 267 |
+
Please see more detailed instructions here: [baselines/pi05](baselines/pi05/README.md).
|
| 268 |
+
|
| 269 |
+
### InternVLA-M1
|
| 270 |
+
Install the env
|
| 271 |
+
```bash
|
| 272 |
+
cd src/InternVLA-M1; uv sync --python 3.10
|
| 273 |
+
```
|
| 274 |
+
1. training
|
| 275 |
+
```bash
|
| 276 |
+
cd src/InternVLA-M1
|
| 277 |
+
bash scripts/train_internvla.sh
|
| 278 |
+
```
|
| 279 |
+
2. serving a checkpoint
|
| 280 |
+
```bash
|
| 281 |
+
cd src/InternVLA-M1
|
| 282 |
+
./scripts/deploy_internvla.sh
|
| 283 |
+
```
|
| 284 |
+
|
| 285 |
+
## Simulation
|
| 286 |
+
|
| 287 |
+
We use [SIMPLE]() to benchmark $\Psi_0$ and all the baselines.
|
| 288 |
+
|
| 289 |
+
> 📢 SIMPLE is an easy-to-use humanoid benchmarking simulator built on the MuJoCo physics engine and Isaac Sim rendering.
|
| 290 |
+
|
| 291 |
+
### Install SIMPLE
|
| 292 |
+
[Coming soon]
|
| 293 |
+
|
| 294 |
+
### Data Generation
|
| 295 |
+
> 📂 We also provide 5 pre-collected whole-body humanoid loco-manipulation tasks at [Huggingface psi-data](https://huggingface.co/datasets/USC-PSI-Lab/psi-data/tree/main/real). If you want to use the existing simulation data, jump to the [Fine-Tuning](#training-sim)
|
| 296 |
+
|
| 297 |
+
#### Motion-Planning Based Data Generation
|
| 298 |
+
[Coming soon]
|
| 299 |
+
|
| 300 |
+
#### Teleoperation in Simulator
|
| 301 |
+
[Coming soon]
|
| 302 |
+
|
| 303 |
+
<a id="training-sim"></a>
|
| 304 |
+
### Fine-Tuning
|
| 305 |
+
|
| 306 |
+
Download [SIMPLE task data](https://huggingface.co/datasets/USC-PSI-Lab/psi-data/tree/main/simple) and extract it:
|
| 307 |
+
|
| 308 |
+
> 💡 Dont forget `source .env` first before following below commands.
|
| 309 |
+
|
| 310 |
+
```
|
| 311 |
+
export task=G1WholebodyBendPick-v0
|
| 312 |
+
|
| 313 |
+
hf download USC-PSI-Lab/psi-data \
|
| 314 |
+
simple/$task$.zip \
|
| 315 |
+
--local-dir=$PSI_HOME/data \
|
| 316 |
+
--repo-type=dataset
|
| 317 |
+
|
| 318 |
+
unzip $PSI_HOME/data/simple/$task.zip -d $PSI_HOME/data/simple
|
| 319 |
+
```
|
| 320 |
+
|
| 321 |
+
> 👀 If you want to visualize the episode please refer to the [Data Visualization](examples/visualize.md) in the examples.
|
| 322 |
+
|
| 323 |
+
Start training:
|
| 324 |
+
|
| 325 |
+
> Please [set up the envrionment variables](#training-real) if not done so yet.
|
| 326 |
+
|
| 327 |
+
```
|
| 328 |
+
bash scripts/train/psi0/finetune-simple-psi0.sh $task
|
| 329 |
+
```
|
| 330 |
+
The training will create a run dir which is located under `.runs` in the project root.
|
| 331 |
+
If your GPU has limited VRAM, set `--train.optimizer-foreach=false` to reduce optimizer-step memory usage at the cost of some speed.
|
| 332 |
+
|
| 333 |
+
### Evaluation in SIMPLE
|
| 334 |
+
|
| 335 |
+
#### Serve $\Psi_0$
|
| 336 |
+
```
|
| 337 |
+
export run_dir=<the run dir here under folder .runs>
|
| 338 |
+
export ckpt_step=<checkpoint step>
|
| 339 |
+
uv run --active --group psi --group serve serve_psi0 \
|
| 340 |
+
--host 0.0.0.0 \
|
| 341 |
+
--port 22085 \
|
| 342 |
+
--run-dir=$run_dir \
|
| 343 |
+
--ckpt-step=$ckpt_step \
|
| 344 |
+
--action-exec-horizon=24 \
|
| 345 |
+
--rtc
|
| 346 |
+
```
|
| 347 |
+
|
| 348 |
+
|
| 349 |
+
Run open-loop evaluation (offline)
|
| 350 |
+
|
| 351 |
+
[examples/simple/openloop_eval.ipynb](examples/simple/openloop_eval.ipynb)
|
| 352 |
+
|
| 353 |
+
#### Run the Evaluation Client
|
| 354 |
+
> If the server is started on a remote server, run ssh port forward. eg., `ssh -L 22086:localhost:22086 songlin@nebula100`.
|
| 355 |
+
|
| 356 |
+
> Once port forward is done, open a new terminal to test if server is up `curl -i http://localhost:22085/health`
|
| 357 |
+
|
| 358 |
+
Launch the eval client through docker
|
| 359 |
+
```
|
| 360 |
+
GPUs=1 docker compose run eval $task psi0 \
|
| 361 |
+
--host=localhost \
|
| 362 |
+
--port=22085 \
|
| 363 |
+
--sim-mode=mujoco_isaac \
|
| 364 |
+
--headless \
|
| 365 |
+
--max-episode-steps=360 \
|
| 366 |
+
--num-episodes=10 \
|
| 367 |
+
--data-format=lerobot \
|
| 368 |
+
--data-dir=data/$task
|
| 369 |
+
```
|
| 370 |
+
The policy rollout videos will be found in folder `third_party/SIMPLE/data/evals/psi0`.
|
| 371 |
+
|
| 372 |
+
> The evaluation for a single episode could take up to 6~10 minutes because SIMPLE use a synchronous rendering API in IsaacSim. See here for [more explanation](#).
|
| 373 |
+
|
| 374 |
+
<a id="pre-post-train"></a>
|
| 375 |
+
## Reproduce Ψ₀: Pre-Training and Post-Training
|
| 376 |
+
|
| 377 |
+
|
| 378 |
+
### Pre-Train VLM
|
| 379 |
+
|
| 380 |
+
Download and cache the official `Qwen/Qwen3-VL-2B-Instruct` weights.
|
| 381 |
+
```
|
| 382 |
+
scripts/predownload_qwen3vl.py
|
| 383 |
+
```
|
| 384 |
+
|
| 385 |
+
Pre-train on the [EgoDex dataset](https://github.com/apple/ml-egodex)
|
| 386 |
+
|
| 387 |
+
Pre-compute `48 DoF EgoDex action`:
|
| 388 |
+
|
| 389 |
+
> We re-use the pre-process code from [H-RDT EgoDex Pre-Processing](https://github.com/HongzheBi/H_RDT?tab=readme-ov-file#data-preprocessing).
|
| 390 |
+
> 1. Change the paths in `src/h_rdt/datasets/pretrain/setup_pretrain.sh`.
|
| 391 |
+
> 2. Tweak the `NUM_PROCESSES` if on a powerful server, i tried max 64.
|
| 392 |
+
> 3. set `FORCE_OVERWRITE=True` if the processing script is disrupted.
|
| 393 |
+
|
| 394 |
+
```
|
| 395 |
+
source src/h_rdt/datasets/pretrain/setup_pretrain.sh
|
| 396 |
+
source .venv-psi/bin/activate
|
| 397 |
+
bash src/h_rdt/datasets/pretrain/run_pretrain_pipeline.sh
|
| 398 |
+
```
|
| 399 |
+
|
| 400 |
+
```
|
| 401 |
+
bash scripts/train/psi0/pretrain-egodex-psi0-fast.sh
|
| 402 |
+
```
|
| 403 |
+
|
| 404 |
+
Pre-train on [humanoid everyday dataset](https://huggingface.co/datasets/USC-GVL/humanoid-everyday)
|
| 405 |
+
|
| 406 |
+
> Please download the pre-processed HE data here: `hf download USC-PSI-Lab/psi-data HE_RAW.zip --repo-type=dataset`
|
| 407 |
+
|
| 408 |
+
```
|
| 409 |
+
bash scripts/train/psi0/pretrain-he-psi0-fast.sh
|
| 410 |
+
```
|
| 411 |
+
|
| 412 |
+
Save the pretrained checkpoints once training is done:
|
| 413 |
+
```
|
| 414 |
+
python scripts/save_pretrain_qwen3vl_backbone.py
|
| 415 |
+
```
|
| 416 |
+
|
| 417 |
+
### Post-Train Action Expert
|
| 418 |
+
|
| 419 |
+
Download pre-trained `psi-0` VLM backbone
|
| 420 |
+
```
|
| 421 |
+
python scripts/data/download.py \
|
| 422 |
+
--repo-id=USC-PSI-Lab/psi-model \
|
| 423 |
+
--remote-dir=pre.fast.egodex.2512241941/pretrained/ckpt_200000 \
|
| 424 |
+
--local-dir=/hfm/cache/checkpoints/psi0/pre.fast.egodex.2512241941.ckpt200k \
|
| 425 |
+
--repo-type=model
|
| 426 |
+
```
|
| 427 |
+
|
| 428 |
+
Post-train on [humanoid everyday (HE) dataset](https://huggingface.co/datasets/USC-GVL/humanoid-everyday)
|
| 429 |
+
```
|
| 430 |
+
bash scripts/train/psi0/posttrain-he-psi0.sh
|
| 431 |
+
```
|
| 432 |
+
|
| 433 |
+
Save post-trained action header once training is over
|
| 434 |
+
```
|
| 435 |
+
python scripts/save_posttrain_action_expert.py
|
| 436 |
+
```
|
| 437 |
+
|
| 438 |
+
## Checkpoints
|
| 439 |
+
|
| 440 |
+
The released checkpoints on [HuggingFace Psi-Model](https://huggingface.co/USC-PSI-Lab/psi-model) is listed
|
| 441 |
+
|
| 442 |
+
| Checkpoint | Description | Remote Directory |
|
| 443 |
+
|---|---|---|
|
| 444 |
+
| $\Psi_0$ VLM<br/>(Baseline) | Pre-trained VLM backbone (EgoDex 200K steps + HE 30K steps) | `psi0/pre.fast.1by1.2601091803.ckpt.ego200k.he30k` |
|
| 445 |
+
| $\Psi_0$ Action Expert<br/>(Baseline) | Post-trained Action Expert On HE | `psi0/postpre.1by1.pad36.2601131206.ckpt.he30k` |
|
| 446 |
+
|
| 447 |
+
and more variants for ablation studies:
|
| 448 |
+
| Checkpoint | Description | Remote Directory |
|
| 449 |
+
|---|---|---|
|
| 450 |
+
| $\Psi_0$ VLM<br/>(Ablation Study) | Pre-trained VLM backbone only on EgoDex 200K steps | `psi0/pre.fast.egodex.2512241941.ckpt200k` |
|
| 451 |
+
| $\Psi_0$ VLM<br/>(Ablation Study) | Pre-trained VLM backbone only on HE 48K steps | `psi0/pre.abl.only.he.2512311516.48k` |
|
| 452 |
+
| $\Psi_0$ VLM<br/>(Ablation Study) | Pre-trained VLM backbone only on 10% EgoDex | `psi0/pre.abl.ego.10per.2602021632.46k` |
|
| 453 |
+
| $\Psi_0$ Action Expert<br/>(Ablation Study) | Post-train on HE by picking pre-trained variant `psi0/pre.abl.only.he.2512311516.48k` | `psi0/postpre.abl.only.he.2602050012` |
|
| 454 |
+
| $\Psi_0$ Action Expert<br/>(Ablation Study) | Post-train on HE by picking pre-trained variant `psi0/pre.abl.ego.10per.2602021632.46k` | `psi0/postpre.abl.ego.10per.2602050006` |
|
| 455 |
+
|
| 456 |
+
|
| 457 |
+
Download the selected models
|
| 458 |
+
|
| 459 |
+
```
|
| 460 |
+
hf download USC-PSI-Lab/psi-model \
|
| 461 |
+
--remote-dir=<remote dictory on huggingface repo>
|
| 462 |
+
--local-dir=$PSI_HOME/cache/checkpoints \
|
| 463 |
+
--repo-type=model
|
| 464 |
+
```
|
| 465 |
+
|
| 466 |
+
## Troubleshootings
|
| 467 |
+
|
| 468 |
+
1. Lerobot dataset issues: `stack(): argument 'tensors' (position 1) must be tuple of Tensors, not Column`
|
| 469 |
+
|
| 470 |
+
This usually means the environment is still on the legacy PSI `lerobot` stack. Resync the PSI env so it uses the
|
| 471 |
+
same `lerobot` and `datasets` versions as SIMPLE, then verify the import layout:
|
| 472 |
+
|
| 473 |
+
```bash
|
| 474 |
+
source .venv-psi/bin/activate
|
| 475 |
+
uv sync --group psi --active
|
| 476 |
+
python -c "from psi.data.lerobot.compat import LEROBOT_LAYOUT; print(LEROBOT_LAYOUT)"
|
| 477 |
+
```
|
| 478 |
+
|
| 479 |
+
2. Fail to install `evdev`, `src/evdev/input.c:10:10: fatal error: Python.h: No such file or directory`
|
| 480 |
+
|
| 481 |
+
```
|
| 482 |
+
sudo apt update
|
| 483 |
+
sudo apt install -y python3-dev python3-venv build-essential \
|
| 484 |
+
linux-headers-$(uname -r)
|
| 485 |
+
```
|
| 486 |
+
|
| 487 |
+
3. RuntimeError: Could not load libtorchcodec. Likely causes ...
|
| 488 |
+
```
|
| 489 |
+
sudo apt-get install ffmpeg
|
| 490 |
+
```
|
| 491 |
+
|
| 492 |
+
4. ImportError: cannot import name 'Deprecated' from 'wandb.proto.wandb_telemetry_pb2'
|
| 493 |
+
|
| 494 |
+
re-install `wandb`
|
| 495 |
+
```
|
| 496 |
+
source .venv-pusht/bin/activate
|
| 497 |
+
uv pip uninstall wandb
|
| 498 |
+
uv pip install wandb==0.18.0
|
| 499 |
+
```
|
| 500 |
+
|
| 501 |
+
5. support `sm_120` on newer GPUs like `5090` or `RTX 6000`, UserWarning: Ignoring invalid value for boolean flag CUDA_LAUNCH_BLOCKING: truevalid values are 0 or 1.
|
| 502 |
+
|
| 503 |
+
update `torch` and `flash-attn`
|
| 504 |
+
```
|
| 505 |
+
uv pip install torch==2.7.0 torchvision==0.22.0 torchaudio==2.7.0 --index-url https://download.pytorch.org/whl/cu128
|
| 506 |
+
uv pip install flash-attn --no-build-isolation
|
| 507 |
+
```
|
| 508 |
+
|
| 509 |
+
6. Failed to download and build `lerobot ... `, Use `git lfs logs last` to view the log.
|
| 510 |
+
|
| 511 |
+
```
|
| 512 |
+
GIT_LFS_SKIP_SMUDGE=1 uv ...
|
| 513 |
+
```
|
| 514 |
+
## Citation
|
| 515 |
+
|
| 516 |
+
```
|
| 517 |
+
@misc{wei2026psi0,
|
| 518 |
+
title={$\Psi_0$: An Open Foundation Model Towards Universal Humanoid Loco-Manipulation},
|
| 519 |
+
author={Songlin Wei and Hongyi Jing and Boqian Li and Zhenyu Zhao and Jiageng Mao and Zhenhao Ni and Sicheng He and Jie Liu and Xiawei Liu and Kaidi Kang and Sheng Zang and Weiduo Yuan and Marco Pavone and Di Huang and Yue Wang},
|
| 520 |
+
year={2026},
|
| 521 |
+
eprint={2603.12263},
|
| 522 |
+
archivePrefix={arXiv},
|
| 523 |
+
primaryClass={cs.RO},
|
| 524 |
+
url={https://arxiv.org/abs/2603.12263},
|
| 525 |
+
}
|
| 526 |
+
```
|
| 527 |
+
|
| 528 |
+
## License
|
| 529 |
+
|
| 530 |
+
This project is licensed under the Apache License 2.0.
|
| 531 |
+
|
| 532 |
+
See the [LICENSE](LICENSE) file for details.
|
flake.lock
ADDED
|
@@ -0,0 +1,27 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"nodes": {
|
| 3 |
+
"nixpkgs": {
|
| 4 |
+
"locked": {
|
| 5 |
+
"lastModified": 1772963539,
|
| 6 |
+
"narHash": "sha256-9jVDGZnvCckTGdYT53d/EfznygLskyLQXYwJLKMPsZs=",
|
| 7 |
+
"owner": "NixOS",
|
| 8 |
+
"repo": "nixpkgs",
|
| 9 |
+
"rev": "9dcb002ca1690658be4a04645215baea8b95f31d",
|
| 10 |
+
"type": "github"
|
| 11 |
+
},
|
| 12 |
+
"original": {
|
| 13 |
+
"owner": "NixOS",
|
| 14 |
+
"ref": "nixos-unstable",
|
| 15 |
+
"repo": "nixpkgs",
|
| 16 |
+
"type": "github"
|
| 17 |
+
}
|
| 18 |
+
},
|
| 19 |
+
"root": {
|
| 20 |
+
"inputs": {
|
| 21 |
+
"nixpkgs": "nixpkgs"
|
| 22 |
+
}
|
| 23 |
+
}
|
| 24 |
+
},
|
| 25 |
+
"root": "root",
|
| 26 |
+
"version": 7
|
| 27 |
+
}
|
infer.sh
ADDED
|
@@ -0,0 +1,41 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# <modify_textop_repr> infer脚本固定到v4仓库。
|
| 2 |
+
# 原逻辑:
|
| 3 |
+
# cd /cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/Psi0-yth
|
| 4 |
+
HFD_ROOT=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
|
| 5 |
+
source "$HFD_ROOT/scripts/activate_psi_env.sh"
|
| 6 |
+
cd /cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/Psi0-yth-v18
|
| 7 |
+
export PYTHONPATH=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/Psi0-yth-v18/src
|
| 8 |
+
export CUDA_VISIBLE_DEVICES=0
|
| 9 |
+
pkill -f monitor.py
|
| 10 |
+
|
| 11 |
+
# prompt
|
| 12 |
+
# PYTHONPATH=src python scripts/infer/seed_text_to_motion.py \
|
| 13 |
+
# --run-dir /cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/Psi0-yth/runs/posttrain_v1/posttrain.sg.flow1000.consta.lr1.0e-04.b512.gpus8.2604160750 \
|
| 14 |
+
# --ckpt-step 1300 \
|
| 15 |
+
# --output-dir /cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/Psi0-yth/runs/posttrain_v1/2604160750_v2 \
|
| 16 |
+
# --prompt "walk backward naturally and stop smoothly" \
|
| 17 |
+
# --prompt-motion-name walk_backward_stop \
|
| 18 |
+
# --num-output-frames 400 \
|
| 19 |
+
# --num-inference-steps 10 \
|
| 20 |
+
# --device cuda:0
|
| 21 |
+
|
| 22 |
+
# metadata
|
| 23 |
+
# RUN_DIR="/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/envs/.psi-home/runs/posttrain_v1/posttrain.sg.flow1000.consta.lr1.0e-04.b512.gpus8.2604161308"
|
| 24 |
+
# OUTPUT_DIR="/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/Psi0-yth/runs/posttrain_v1/2604161306"
|
| 25 |
+
# RUN_DIR="/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/envs/.psi-home/runs/posttrain_v1/posttrain.sg.flow1000.consta.lr1.0e-04.b512.gpus8.2604161308"
|
| 26 |
+
# OUTPUT_DIR="/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/Psi0-yth/runs/posttrain_v1/2604161308"
|
| 27 |
+
RUN_DIR="/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/envs/.psi-home/runs/posttrain_v1/posttrain.sg.flow1000.consta.lr1.0e-04.b512.gpus8.2604170218"
|
| 28 |
+
# <modify_states> 默认infer输出改到v2仓库。
|
| 29 |
+
# 原逻辑:
|
| 30 |
+
# OUTPUT_DIR="/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/Psi0-yth/runs/posttrain_v1/2604170218"
|
| 31 |
+
OUTPUT_DIR="/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/Psi0-yth-v18/runs/posttrain_v1/2604170218"
|
| 32 |
+
# RUN_DIR="/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/envs/.psi-home/runs/posttrain_v1/posttrain.sg.flow1000.consta.lr1.0e-04.b512.gpus8.2604170323"
|
| 33 |
+
# OUTPUT_DIR="/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/Psi0-yth/runs/posttrain_v1/2604170323"
|
| 34 |
+
python scripts/infer/seed_text_to_motion.py \
|
| 35 |
+
--run-dir $RUN_DIR \
|
| 36 |
+
--ckpt-step 55000 \
|
| 37 |
+
--output-dir $OUTPUT_DIR \
|
| 38 |
+
--num-samples 1 \
|
| 39 |
+
--start-index 8 \
|
| 40 |
+
--num-inference-steps 10 \
|
| 41 |
+
--device cuda:0
|
infer_prompt.sh
ADDED
|
@@ -0,0 +1,214 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
HFD_ROOT=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
|
| 2 |
+
source "$HFD_ROOT/scripts/activate_psi_env.sh"
|
| 3 |
+
cd /cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/Psi0-yth-v18
|
| 4 |
+
[ -f .env ] && source .env
|
| 5 |
+
|
| 6 |
+
export PYTHONPATH=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/Psi0-yth-v18/src
|
| 7 |
+
export CUDA_VISIBLE_DEVICES=0
|
| 8 |
+
export PSI_INFER_FLOAT32=1
|
| 9 |
+
pkill -f monitor.py
|
| 10 |
+
|
| 11 |
+
# 改这里选择推理类型:audio | audio_command | audio_command_only | text | video | video_clip
|
| 12 |
+
MODE=video_clip
|
| 13 |
+
|
| 14 |
+
NUM_INFERENCE_STEPS=10
|
| 15 |
+
DEVICE=cuda:0
|
| 16 |
+
|
| 17 |
+
RUN_TIMESTAMP=2606041200
|
| 18 |
+
# RUN_TIMESTAMP=2606041447
|
| 19 |
+
RUN_PARENT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/Psi0-yth-v18/runs/posttrain_v1_trimodal
|
| 20 |
+
RUN_DIR=$RUN_PARENT/posttrain.sg.flow1000.cwml.lr1.0e-04.b256.gpus8.$RUN_TIMESTAMP
|
| 21 |
+
CKPT_STEP=best
|
| 22 |
+
|
| 23 |
+
# 普通 audio / video 默认样例
|
| 24 |
+
MOTION_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/GR00T-WholeBodyControl/gear_sonic_deploy/reference/converted/psi0_dance/gvhmr
|
| 25 |
+
VIDEO_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/dance_1723_video_psi0/dance_1723_video_10s
|
| 26 |
+
MOTION_NAME=movies__F-SNBR4jhY_0007
|
| 27 |
+
# MOTION_DIR=$MOTION_ROOT/$MOTION_NAME
|
| 28 |
+
# VIDEO_PATH=$VIDEO_ROOT/$MOTION_NAME.mp4
|
| 29 |
+
# VIDEO_PATH=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/videogen/motion_example_0506_demo/original_videos/00000/0h7AlrKLslw_00056_0_313.mp4
|
| 30 |
+
# VIDEO_PATH=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/dance_1723_video_psi0/dance_1723_video_10s/movies__F-SNBR4jhY_0007.mp4
|
| 31 |
+
|
| 32 |
+
MOTION_DIR=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/GR00T-WholeBodyControl/gear_sonic_deploy/reference/converted/psi0_dance/gvhmr/movies__6RU6bQLqVk_0000
|
| 33 |
+
VIDEO_PATH=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/Psi0-yth-v18/kun3.mp4
|
| 34 |
+
|
| 35 |
+
# MOTION_DIR=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/videogen/psi0_video_clip_50f_rand8/dance_1723_video_10s/motion/movies__F-SNBR4jhY_0007.chunk00.f000160
|
| 36 |
+
# VIDEO_PATH=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/videogen/psi0_video_clip_50f_rand8/dance_1723_video_10s/videos/movies__F-SNBR4jhY_0007.chunk00.f000160.mp4
|
| 37 |
+
|
| 38 |
+
# MOTION_DIR=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/videogen/psi0_video_clip_50f_rand32_full/motion_example_0506_demo/motion/converted_motion__00000___dFAYYMJbWk_00193_268_382.chunk00.f000000
|
| 39 |
+
# VIDEO_PATH=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/videogen/psi0_video_clip_50f_rand32_full/motion_example_0506_demo/videos/converted_motion__00000___dFAYYMJbWk_00193_268_382.chunk00.f000000.mp4
|
| 40 |
+
|
| 41 |
+
# audio_command 默认样例;AUDIO_COMMAND_AUDIO_PATH 是目录时会随机抽一条 wav
|
| 42 |
+
# AUDIO_COMMAND_MOTION_DIR=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/Psi0/dataset/seed_all/g1/csv_sonic/230530/sitting_legs_bend_arms_front_loop_R_001__A410
|
| 43 |
+
# AUDIO_COMMAND_AUDIO_PATH=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/qwen3-tts/seed_audio/003856
|
| 44 |
+
# AUDIO_COMMAND_AUDIO_PATH=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/qwen3-tts/seed_audio/000017
|
| 45 |
+
AUDIO_COMMAND_MOTION_DIR=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/GR00T-WholeBodyControl/gear_sonic_deploy/reference/converted/psi0_dance/gvhmr/movies_1mzl0Zh614A_0000
|
| 46 |
+
AUDIO_COMMAND_PROMPT="Generate a motion that follows the spoken instruction in the audio"
|
| 47 |
+
AUDIO_COMMAND_AUDIO_PATH=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/Psi0-yth-v18/jump.wav
|
| 48 |
+
|
| 49 |
+
# text 默认样例
|
| 50 |
+
TEXT_PROMPT="Generate the following motion sequence: A person jog forward."
|
| 51 |
+
TEXT_START_INDEX=0
|
| 52 |
+
TEXT_NUM_OUTPUT_FRAMES=300
|
| 53 |
+
PROMPT_MOTION_NAME=text_prompt
|
| 54 |
+
|
| 55 |
+
case "$MODE" in
|
| 56 |
+
audio|audio_text|audio+text|audio_only|audio_pure)
|
| 57 |
+
MODE_NAME=audio
|
| 58 |
+
AUDIO_CONFIG_KEY=audio
|
| 59 |
+
OUTPUT_DIR=$RUN_PARENT/$RUN_TIMESTAMP.$MODE_NAME.test
|
| 60 |
+
PROMPT="Generate a dance sequence that matches the provided music."
|
| 61 |
+
if [ "$MODE" = "audio_only" ] || [ "$MODE" = "audio_pure" ]; then
|
| 62 |
+
PROMPT=""
|
| 63 |
+
fi
|
| 64 |
+
|
| 65 |
+
# rm -rf "$OUTPUT_DIR"
|
| 66 |
+
|
| 67 |
+
echo "Mode: $MODE_NAME"
|
| 68 |
+
echo "Audio config key: $AUDIO_CONFIG_KEY"
|
| 69 |
+
echo "Run dir: $RUN_DIR"
|
| 70 |
+
echo "Ckpt step: $CKPT_STEP"
|
| 71 |
+
echo "Prompt: $PROMPT"
|
| 72 |
+
echo "Motion dir: $MOTION_DIR"
|
| 73 |
+
echo "Video path: $VIDEO_PATH"
|
| 74 |
+
echo "Output dir: $OUTPUT_DIR"
|
| 75 |
+
|
| 76 |
+
python scripts/infer/seed_audio_to_motion.py \
|
| 77 |
+
--run-dir "$RUN_DIR" \
|
| 78 |
+
--ckpt-step "$CKPT_STEP" \
|
| 79 |
+
--output-dir "$OUTPUT_DIR" \
|
| 80 |
+
--motion-dir "$MOTION_DIR" \
|
| 81 |
+
--video-path "$VIDEO_PATH" \
|
| 82 |
+
--audio-config-key "$AUDIO_CONFIG_KEY" \
|
| 83 |
+
--prompt "$PROMPT" \
|
| 84 |
+
--num-inference-steps "$NUM_INFERENCE_STEPS" \
|
| 85 |
+
--device "$DEVICE" || return 1 2>/dev/null || exit 1
|
| 86 |
+
;;
|
| 87 |
+
audio_command|audio_command_text|audio_command_only|command_audio)
|
| 88 |
+
MODE_NAME=audio_command
|
| 89 |
+
AUDIO_CONFIG_KEY=audio_command
|
| 90 |
+
OUTPUT_DIR=$RUN_PARENT/$RUN_TIMESTAMP.$MODE_NAME.test
|
| 91 |
+
MOTION_DIR=$AUDIO_COMMAND_MOTION_DIR
|
| 92 |
+
VIDEO_PATH=$AUDIO_COMMAND_AUDIO_PATH
|
| 93 |
+
PROMPT=$AUDIO_COMMAND_PROMPT
|
| 94 |
+
if [ "$MODE" = "audio_command_only" ]; then
|
| 95 |
+
PROMPT=""
|
| 96 |
+
fi
|
| 97 |
+
|
| 98 |
+
# rm -rf "$OUTPUT_DIR"
|
| 99 |
+
|
| 100 |
+
echo "Mode: $MODE_NAME"
|
| 101 |
+
echo "Audio config key: $AUDIO_CONFIG_KEY"
|
| 102 |
+
echo "Run dir: $RUN_DIR"
|
| 103 |
+
echo "Ckpt step: $CKPT_STEP"
|
| 104 |
+
echo "Prompt: $PROMPT"
|
| 105 |
+
echo "Motion dir: $MOTION_DIR"
|
| 106 |
+
echo "Video path: $VIDEO_PATH"
|
| 107 |
+
echo "Output dir: $OUTPUT_DIR"
|
| 108 |
+
|
| 109 |
+
python scripts/infer/seed_audio_to_motion.py \
|
| 110 |
+
--run-dir "$RUN_DIR" \
|
| 111 |
+
--ckpt-step "$CKPT_STEP" \
|
| 112 |
+
--output-dir "$OUTPUT_DIR" \
|
| 113 |
+
--motion-dir "$MOTION_DIR" \
|
| 114 |
+
--video-path "$VIDEO_PATH" \
|
| 115 |
+
--audio-config-key "$AUDIO_CONFIG_KEY" \
|
| 116 |
+
--prompt "$PROMPT" \
|
| 117 |
+
--num-inference-steps "$NUM_INFERENCE_STEPS" \
|
| 118 |
+
--device "$DEVICE" || return 1 2>/dev/null || exit 1
|
| 119 |
+
;;
|
| 120 |
+
text|pure_text)
|
| 121 |
+
MODE_NAME=text
|
| 122 |
+
OUTPUT_DIR=$RUN_PARENT/$RUN_TIMESTAMP.$MODE_NAME.test
|
| 123 |
+
|
| 124 |
+
# rm -rf "$OUTPUT_DIR"
|
| 125 |
+
|
| 126 |
+
echo "Mode: $MODE_NAME"
|
| 127 |
+
echo "Run dir: $RUN_DIR"
|
| 128 |
+
echo "Ckpt step: $CKPT_STEP"
|
| 129 |
+
echo "Prompt: $TEXT_PROMPT"
|
| 130 |
+
echo "Prompt motion name: $PROMPT_MOTION_NAME"
|
| 131 |
+
echo "Start index: $TEXT_START_INDEX"
|
| 132 |
+
echo "Num output frames: $TEXT_NUM_OUTPUT_FRAMES"
|
| 133 |
+
echo "Output dir: $OUTPUT_DIR"
|
| 134 |
+
|
| 135 |
+
python scripts/infer/seed_text_to_motion.py \
|
| 136 |
+
--run-dir "$RUN_DIR" \
|
| 137 |
+
--ckpt-step "$CKPT_STEP" \
|
| 138 |
+
--output-dir "$OUTPUT_DIR" \
|
| 139 |
+
--prompt "$TEXT_PROMPT" \
|
| 140 |
+
--prompt-motion-name "$PROMPT_MOTION_NAME" \
|
| 141 |
+
--init-motion-dir "$MOTION_DIR" \
|
| 142 |
+
--start-index "$TEXT_START_INDEX" \
|
| 143 |
+
--num-output-frames "$TEXT_NUM_OUTPUT_FRAMES" \
|
| 144 |
+
--num-inference-steps "$NUM_INFERENCE_STEPS" \
|
| 145 |
+
--force-text-only \
|
| 146 |
+
--device "$DEVICE" || return 1 2>/dev/null || exit 1
|
| 147 |
+
;;
|
| 148 |
+
video|video_text|video+text|video_only|video_pure|video_clip|video_clip_text|video_clip_only)
|
| 149 |
+
if [ "$MODE" = "video_clip" ] || [ "$MODE" = "video_clip_text" ] || [ "$MODE" = "video_clip_only" ]; then
|
| 150 |
+
MODE_NAME=video_clip
|
| 151 |
+
VIDEO_CONFIG_KEY=video_clip
|
| 152 |
+
VIDEO_CLIP_ARGS=(--video-config-key video_clip --video-clip)
|
| 153 |
+
else
|
| 154 |
+
MODE_NAME=video
|
| 155 |
+
VIDEO_CONFIG_KEY=video
|
| 156 |
+
VIDEO_CLIP_ARGS=(--video-config-key video)
|
| 157 |
+
fi
|
| 158 |
+
OUTPUT_DIR=$RUN_PARENT/$RUN_TIMESTAMP.$MODE_NAME.test
|
| 159 |
+
PROMPT="Generate a motion sequence that matches the provided video."
|
| 160 |
+
if [ "$MODE" = "video_only" ] || [ "$MODE" = "video_pure" ] || [ "$MODE" = "video_clip_only" ]; then
|
| 161 |
+
PROMPT=""
|
| 162 |
+
fi
|
| 163 |
+
|
| 164 |
+
# rm -rf "$OUTPUT_DIR"
|
| 165 |
+
|
| 166 |
+
echo "Mode: $MODE_NAME"
|
| 167 |
+
echo "Video config key: $VIDEO_CONFIG_KEY"
|
| 168 |
+
echo "Run dir: $RUN_DIR"
|
| 169 |
+
echo "Ckpt step: $CKPT_STEP"
|
| 170 |
+
echo "Prompt: $PROMPT"
|
| 171 |
+
echo "Motion dir: $MOTION_DIR"
|
| 172 |
+
echo "Video path: $VIDEO_PATH"
|
| 173 |
+
echo "Output dir: $OUTPUT_DIR"
|
| 174 |
+
|
| 175 |
+
python scripts/infer/seed_video_to_motion.py \
|
| 176 |
+
--run-dir "$RUN_DIR" \
|
| 177 |
+
--ckpt-step "$CKPT_STEP" \
|
| 178 |
+
--output-dir "$OUTPUT_DIR" \
|
| 179 |
+
--motion-dir "$MOTION_DIR" \
|
| 180 |
+
--video-path "$VIDEO_PATH" \
|
| 181 |
+
--prompt "$PROMPT" \
|
| 182 |
+
--num-inference-steps "$NUM_INFERENCE_STEPS" \
|
| 183 |
+
"${VIDEO_CLIP_ARGS[@]}" \
|
| 184 |
+
--device "$DEVICE" || return 1 2>/dev/null || exit 1
|
| 185 |
+
;;
|
| 186 |
+
*)
|
| 187 |
+
echo "ERROR: unknown mode '$MODE'. Use 'audio', 'audio_command', 'audio_command_only', 'text', 'video', or 'video_clip'."
|
| 188 |
+
exit 1
|
| 189 |
+
;;
|
| 190 |
+
esac
|
| 191 |
+
|
| 192 |
+
deactivate
|
| 193 |
+
unset PYTHONPATH
|
| 194 |
+
cd /cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/GR00T-WholeBodyControl/gear_sonic
|
| 195 |
+
export PYOPENGL_PLATFORM="egl"
|
| 196 |
+
export MUJOCO_GL="egl"
|
| 197 |
+
export QT_X11_NO_MITSHM=1
|
| 198 |
+
xvfb-run -a -s "-screen 0 1280x1024x24 -extension MIT-SHM" /cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/envs/gmr/bin/python scripts/batch_render_reference_motion_mujoco.py \
|
| 199 |
+
--motion-dir "$OUTPUT_DIR" \
|
| 200 |
+
--source-fps 50 \
|
| 201 |
+
--target-fps 30 \
|
| 202 |
+
--dof-order isaaclab
|
| 203 |
+
|
| 204 |
+
# deactivate
|
| 205 |
+
# unset PYTHONPATH
|
| 206 |
+
# cd /cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/GR00T-WholeBodyControl/gear_sonic
|
| 207 |
+
# export PYOPENGL_PLATFORM="egl"
|
| 208 |
+
# export MUJOCO_GL="egl"
|
| 209 |
+
# export QT_X11_NO_MITSHM=1
|
| 210 |
+
# xvfb-run -a -s "-screen 0 1280x1024x24 -extension MIT-SHM" /cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/envs/gmr/bin/python scripts/batch_render_reference_motion_mujoco.py \
|
| 211 |
+
# --motion-dir "/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/videogen/psi0_video_clip_50f_rand32_full/motion_example_0506_demo/motion/converted_motion__00000___dFAYYMJbWk_00193_268_382.chunk00.f000000" \
|
| 212 |
+
# --source-fps 50 \
|
| 213 |
+
# --target-fps 30 \
|
| 214 |
+
# --dof-order isaaclab
|
infer_prompt_distill.sh
ADDED
|
@@ -0,0 +1,105 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#!/usr/bin/env bash
|
| 2 |
+
pkill -f monitor.py
|
| 3 |
+
|
| 4 |
+
############################
|
| 5 |
+
# Edit config only here.
|
| 6 |
+
############################
|
| 7 |
+
|
| 8 |
+
STAGE=student
|
| 9 |
+
|
| 10 |
+
# RUN_DIR=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/Psi0-yth-v18/runs/posttrain_v1_motion_distill/posttrain.motion_distill_teacher.sg.flow1000.cwml.lr1.0e-04.b512.gpus8.2605302159
|
| 11 |
+
RUN_DIR=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/Psi0-yth-v18/runs/posttrain_v1_motion_distill/posttrain.motion_distill_student.sg.flow1000.cwml.lr1.0e-04.b256.gpus8.2605310757
|
| 12 |
+
CKPT_STEP=best
|
| 13 |
+
# OUTPUT_DIR=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/Psi0-yth-v18/runs/posttrain_v1_motion_distill/teacher_reconstruct.test
|
| 14 |
+
OUTPUT_DIR=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/Psi0-yth-v18/runs/posttrain_v1_motion_distill/student_infer.test
|
| 15 |
+
|
| 16 |
+
MOTION_DIR=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/videogen/psi0_video_clip_50f_rand8/dance_1723_video_10s/motion/movies__F-SNBR4jhY_0007.chunk00.f000160
|
| 17 |
+
VIDEO_PATH=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/videogen/psi0_video_clip_50f_rand8/dance_1723_video_10s/videos/movies__F-SNBR4jhY_0007.chunk00.f000160.mp4
|
| 18 |
+
PROMPT="Generate a motion sequence that matches the provided video."
|
| 19 |
+
|
| 20 |
+
NUM_OUTPUT_FRAMES=0
|
| 21 |
+
NUM_INFERENCE_STEPS=10
|
| 22 |
+
DEVICE=cuda:0
|
| 23 |
+
CUDA_VISIBLE_DEVICES=0
|
| 24 |
+
PSI_INFER_FLOAT32=1
|
| 25 |
+
|
| 26 |
+
VIDEO_CONFIG_KEY=video_clip
|
| 27 |
+
VIDEO_CLIP=1
|
| 28 |
+
RENDER=1
|
| 29 |
+
|
| 30 |
+
REPO_DIR=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/Psi0-yth-v18
|
| 31 |
+
HFD_ROOT=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
|
| 32 |
+
GEAR_SONIC_DIR=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/GR00T-WholeBodyControl/gear_sonic
|
| 33 |
+
GMR_PYTHON=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/envs/gmr/bin/python
|
| 34 |
+
|
| 35 |
+
############################
|
| 36 |
+
|
| 37 |
+
cd "$REPO_DIR"
|
| 38 |
+
|
| 39 |
+
source "$HFD_ROOT/scripts/activate_psi_env.sh"
|
| 40 |
+
[ -f .env ] && source .env
|
| 41 |
+
|
| 42 |
+
export PYTHONPATH=$REPO_DIR/src
|
| 43 |
+
export CUDA_VISIBLE_DEVICES
|
| 44 |
+
export PSI_INFER_FLOAT32
|
| 45 |
+
|
| 46 |
+
pkill -f monitor.py 2>/dev/null || true
|
| 47 |
+
rm -rf "$OUTPUT_DIR"
|
| 48 |
+
|
| 49 |
+
echo "Stage: $STAGE"
|
| 50 |
+
echo "Run dir: $RUN_DIR"
|
| 51 |
+
echo "Ckpt step: $CKPT_STEP"
|
| 52 |
+
echo "Motion dir: $MOTION_DIR"
|
| 53 |
+
echo "Video path: $VIDEO_PATH"
|
| 54 |
+
echo "Output dir: $OUTPUT_DIR"
|
| 55 |
+
echo "Num output frames: $NUM_OUTPUT_FRAMES"
|
| 56 |
+
echo "Num inference steps: $NUM_INFERENCE_STEPS"
|
| 57 |
+
echo "Device: $DEVICE"
|
| 58 |
+
|
| 59 |
+
if [ "$STAGE" = "teacher" ]; then
|
| 60 |
+
python scripts/infer/seed_motion_teacher_reconstruct.py \
|
| 61 |
+
--run-dir "$RUN_DIR" \
|
| 62 |
+
--ckpt-step "$CKPT_STEP" \
|
| 63 |
+
--output-dir "$OUTPUT_DIR" \
|
| 64 |
+
--motion-dir "$MOTION_DIR" \
|
| 65 |
+
--num-output-frames "$NUM_OUTPUT_FRAMES" \
|
| 66 |
+
--num-inference-steps "$NUM_INFERENCE_STEPS" \
|
| 67 |
+
--device "$DEVICE"
|
| 68 |
+
elif [ "$STAGE" = "student" ]; then
|
| 69 |
+
VIDEO_ARGS=(--video-config-key "$VIDEO_CONFIG_KEY")
|
| 70 |
+
if [ "$VIDEO_CLIP" = "1" ]; then
|
| 71 |
+
VIDEO_ARGS+=(--video-clip)
|
| 72 |
+
fi
|
| 73 |
+
|
| 74 |
+
python scripts/infer/seed_video_to_motion.py \
|
| 75 |
+
--run-dir "$RUN_DIR" \
|
| 76 |
+
--ckpt-step "$CKPT_STEP" \
|
| 77 |
+
--output-dir "$OUTPUT_DIR" \
|
| 78 |
+
--motion-dir "$MOTION_DIR" \
|
| 79 |
+
--video-path "$VIDEO_PATH" \
|
| 80 |
+
--prompt "$PROMPT" \
|
| 81 |
+
--num-output-frames "$NUM_OUTPUT_FRAMES" \
|
| 82 |
+
--num-inference-steps "$NUM_INFERENCE_STEPS" \
|
| 83 |
+
"${VIDEO_ARGS[@]}" \
|
| 84 |
+
--device "$DEVICE"
|
| 85 |
+
else
|
| 86 |
+
echo "ERROR: STAGE must be teacher or student."
|
| 87 |
+
exit 1
|
| 88 |
+
fi
|
| 89 |
+
|
| 90 |
+
deactivate
|
| 91 |
+
unset PYTHONPATH
|
| 92 |
+
|
| 93 |
+
if [ "$RENDER" != "1" ]; then
|
| 94 |
+
exit 0
|
| 95 |
+
fi
|
| 96 |
+
|
| 97 |
+
cd "$GEAR_SONIC_DIR"
|
| 98 |
+
export PYOPENGL_PLATFORM=egl
|
| 99 |
+
export MUJOCO_GL=egl
|
| 100 |
+
export QT_X11_NO_MITSHM=1
|
| 101 |
+
xvfb-run -a -s "-screen 0 1280x1024x24 -extension MIT-SHM" "$GMR_PYTHON" scripts/batch_render_reference_motion_mujoco.py \
|
| 102 |
+
--motion-dir "$OUTPUT_DIR" \
|
| 103 |
+
--source-fps 50 \
|
| 104 |
+
--target-fps 30 \
|
| 105 |
+
--dof-order isaaclab
|
infer_prompt_short.sh
ADDED
|
@@ -0,0 +1,158 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#!/usr/bin/env bash
|
| 2 |
+
set -euo pipefail
|
| 3 |
+
|
| 4 |
+
HFD_ROOT=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
|
| 5 |
+
source "$HFD_ROOT/scripts/activate_psi_env.sh"
|
| 6 |
+
cd /cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/Psi0-yth-v18
|
| 7 |
+
[ -f .env ] && source .env
|
| 8 |
+
|
| 9 |
+
export PYTHONPATH=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/Psi0-yth-v18/src
|
| 10 |
+
export CUDA_VISIBLE_DEVICES=0
|
| 11 |
+
export PSI_INFER_FLOAT32=1
|
| 12 |
+
pkill -f monitor.py || true
|
| 13 |
+
|
| 14 |
+
# 改这里:video2motion 只 rollout 多少个实际执行 chunk。
|
| 15 |
+
SHORT_NUM_CHUNKS=2
|
| 16 |
+
# 改这里选择短推理类型:video | video_only | video_clip | video_clip_only
|
| 17 |
+
MODE=video_clip
|
| 18 |
+
|
| 19 |
+
NUM_INFERENCE_STEPS=10
|
| 20 |
+
DEVICE=cuda:0
|
| 21 |
+
|
| 22 |
+
RUN_TIMESTAMP=2605211455
|
| 23 |
+
RUN_PARENT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/Psi0-yth-v18/runs/posttrain_v1_trimodal
|
| 24 |
+
RUN_DIR=$RUN_PARENT/posttrain.sg.flow1000.cwml.lr1.0e-04.b512.gpus8.$RUN_TIMESTAMP
|
| 25 |
+
CKPT_STEP=best
|
| 26 |
+
|
| 27 |
+
MOTION_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/GR00T-WholeBodyControl/gear_sonic_deploy/reference/converted/psi0_dance/gvhmr
|
| 28 |
+
VIDEO_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/dance_1723_video_psi0/dance_1723_video_10s
|
| 29 |
+
MOTION_NAME=movies_1mzl0Zh614A_0000
|
| 30 |
+
MOTION_DIR=$MOTION_ROOT/$MOTION_NAME
|
| 31 |
+
# VIDEO_PATH=$VIDEO_ROOT/$MOTION_NAME.mp4
|
| 32 |
+
# VIDEO_PATH=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/videogen/motion_example_0506_demo/original_videos/00000/0h7AlrKLslw_00056_0_313.mp4
|
| 33 |
+
VIDEO_PATH=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/dance_1723_video_psi0/dance_1723_video_10s/movies__F-SNBR4jhY_0007.mp4
|
| 34 |
+
|
| 35 |
+
PROMPT="Generate a motion sequence that matches the provided video."
|
| 36 |
+
# PROMPT=""
|
| 37 |
+
TARGET_FPS=50
|
| 38 |
+
|
| 39 |
+
if [ "$SHORT_NUM_CHUNKS" -le 0 ]; then
|
| 40 |
+
echo "ERROR: SHORT_NUM_CHUNKS must be positive, got $SHORT_NUM_CHUNKS"
|
| 41 |
+
exit 1
|
| 42 |
+
fi
|
| 43 |
+
|
| 44 |
+
case "$MODE" in
|
| 45 |
+
video|video_text|video+text|video_only|video_pure)
|
| 46 |
+
MODE_NAME=video
|
| 47 |
+
VIDEO_CONFIG_KEY=video
|
| 48 |
+
VIDEO_CLIP_ARGS=(--video-config-key video)
|
| 49 |
+
;;
|
| 50 |
+
video_clip|video_clip_text|video_clip_only)
|
| 51 |
+
MODE_NAME=video_clip
|
| 52 |
+
VIDEO_CONFIG_KEY=video_clip
|
| 53 |
+
VIDEO_CLIP_ARGS=(--video-config-key video_clip --video-clip --video-clip-motion-fps "$TARGET_FPS")
|
| 54 |
+
;;
|
| 55 |
+
*)
|
| 56 |
+
echo "ERROR: unknown mode '$MODE'. Use 'video' or 'video_clip'."
|
| 57 |
+
exit 1
|
| 58 |
+
;;
|
| 59 |
+
esac
|
| 60 |
+
|
| 61 |
+
if [ "$MODE" = "video_only" ] || [ "$MODE" = "video_pure" ] || [ "$MODE" = "video_clip_only" ]; then
|
| 62 |
+
PROMPT=""
|
| 63 |
+
fi
|
| 64 |
+
|
| 65 |
+
RUN_CONFIG=$RUN_DIR/run_config.json
|
| 66 |
+
if [ ! -f "$RUN_CONFIG" ]; then
|
| 67 |
+
echo "ERROR: run_config.json not found: $RUN_CONFIG"
|
| 68 |
+
exit 1
|
| 69 |
+
fi
|
| 70 |
+
|
| 71 |
+
read -r RTC_ENABLED ACTION_HORIZON EXEC_HORIZON <<< "$(python -c 'import json, sys
|
| 72 |
+
cfg = json.load(open(sys.argv[1], encoding="utf-8"))
|
| 73 |
+
model = cfg.get("model", {})
|
| 74 |
+
rtc = bool(model.get("rtc", False))
|
| 75 |
+
action_horizon = int(model.get("action_chunk_size", 30))
|
| 76 |
+
if rtc:
|
| 77 |
+
exec_horizon = int(model.get("action_exec_horizon", action_horizon))
|
| 78 |
+
else:
|
| 79 |
+
exec_horizon = action_horizon
|
| 80 |
+
print(int(rtc), action_horizon, exec_horizon)' "$RUN_CONFIG")"
|
| 81 |
+
NUM_OUTPUT_FRAMES=$((SHORT_NUM_CHUNKS * EXEC_HORIZON))
|
| 82 |
+
if [ "$MODE_NAME" = "video_clip" ]; then
|
| 83 |
+
VIDEO_INPUT_FRAMES=$(((SHORT_NUM_CHUNKS - 1) * EXEC_HORIZON + ACTION_HORIZON))
|
| 84 |
+
else
|
| 85 |
+
VIDEO_INPUT_FRAMES=$NUM_OUTPUT_FRAMES
|
| 86 |
+
fi
|
| 87 |
+
OUTPUT_DIR=$RUN_PARENT/$RUN_TIMESTAMP.$MODE_NAME.short_${SHORT_NUM_CHUNKS}chunks
|
| 88 |
+
VIDEO_DURATION_SEC=$(python -c 'import sys
|
| 89 |
+
frames = int(sys.argv[1])
|
| 90 |
+
fps = float(sys.argv[2])
|
| 91 |
+
print(f"{frames / fps:.6f}")' "$VIDEO_INPUT_FRAMES" "$TARGET_FPS")
|
| 92 |
+
SOURCE_VIDEO_DURATION_SEC=$(ffprobe -v error -show_entries format=duration -of default=nk=1:nw=1 "$VIDEO_PATH")
|
| 93 |
+
python -c 'import sys
|
| 94 |
+
need = float(sys.argv[1])
|
| 95 |
+
have = float(sys.argv[2])
|
| 96 |
+
if have + 1e-3 < need:
|
| 97 |
+
raise SystemExit(f"ERROR: source video is shorter than requested clip: need {need:.6f}s, have {have:.6f}s")' \
|
| 98 |
+
"$VIDEO_DURATION_SEC" "$SOURCE_VIDEO_DURATION_SEC"
|
| 99 |
+
SHORT_INPUT_DIR=$RUN_PARENT/$RUN_TIMESTAMP.$MODE_NAME.short_inputs
|
| 100 |
+
VIDEO_BASENAME=$(basename "$VIDEO_PATH")
|
| 101 |
+
SHORT_VIDEO_PATH=$SHORT_INPUT_DIR/${VIDEO_BASENAME%.*}.short_${SHORT_NUM_CHUNKS}chunks.mp4
|
| 102 |
+
|
| 103 |
+
rm -rf "$OUTPUT_DIR"
|
| 104 |
+
mkdir -p "$SHORT_INPUT_DIR"
|
| 105 |
+
ffmpeg -hide_banner -loglevel error -y \
|
| 106 |
+
-ss 0 \
|
| 107 |
+
-t "$VIDEO_DURATION_SEC" \
|
| 108 |
+
-i "$VIDEO_PATH" \
|
| 109 |
+
-map 0:v:0 \
|
| 110 |
+
-an \
|
| 111 |
+
-c:v libx264 \
|
| 112 |
+
-pix_fmt yuv420p \
|
| 113 |
+
"$SHORT_VIDEO_PATH"
|
| 114 |
+
CROPPED_VIDEO_DURATION_SEC=$(ffprobe -v error -show_entries format=duration -of default=nk=1:nw=1 "$SHORT_VIDEO_PATH")
|
| 115 |
+
|
| 116 |
+
echo "Mode: $MODE_NAME short"
|
| 117 |
+
echo "Video config key: $VIDEO_CONFIG_KEY"
|
| 118 |
+
echo "Short chunks: $SHORT_NUM_CHUNKS"
|
| 119 |
+
echo "RTC enabled: $RTC_ENABLED"
|
| 120 |
+
echo "Action horizon: $ACTION_HORIZON"
|
| 121 |
+
echo "RTC overlap frames: $((ACTION_HORIZON - EXEC_HORIZON))"
|
| 122 |
+
echo "Exec horizon: $EXEC_HORIZON"
|
| 123 |
+
echo "Num output frames: $NUM_OUTPUT_FRAMES"
|
| 124 |
+
echo "Video input frames: $VIDEO_INPUT_FRAMES"
|
| 125 |
+
echo "Target fps: $TARGET_FPS"
|
| 126 |
+
echo "Short video seconds: $VIDEO_DURATION_SEC"
|
| 127 |
+
echo "Cropped video seconds: $CROPPED_VIDEO_DURATION_SEC"
|
| 128 |
+
echo "Run dir: $RUN_DIR"
|
| 129 |
+
echo "Ckpt step: $CKPT_STEP"
|
| 130 |
+
echo "Prompt: $PROMPT"
|
| 131 |
+
echo "Motion dir: $MOTION_DIR"
|
| 132 |
+
echo "Video path: $VIDEO_PATH"
|
| 133 |
+
echo "Short video path: $SHORT_VIDEO_PATH"
|
| 134 |
+
echo "Output dir: $OUTPUT_DIR"
|
| 135 |
+
|
| 136 |
+
python scripts/infer/seed_video_to_motion.py \
|
| 137 |
+
--run-dir "$RUN_DIR" \
|
| 138 |
+
--ckpt-step "$CKPT_STEP" \
|
| 139 |
+
--output-dir "$OUTPUT_DIR" \
|
| 140 |
+
--motion-dir "$MOTION_DIR" \
|
| 141 |
+
--video-path "$SHORT_VIDEO_PATH" \
|
| 142 |
+
--prompt "$PROMPT" \
|
| 143 |
+
--num-output-frames "$NUM_OUTPUT_FRAMES" \
|
| 144 |
+
--num-inference-steps "$NUM_INFERENCE_STEPS" \
|
| 145 |
+
"${VIDEO_CLIP_ARGS[@]}" \
|
| 146 |
+
--device "$DEVICE"
|
| 147 |
+
|
| 148 |
+
deactivate
|
| 149 |
+
unset PYTHONPATH
|
| 150 |
+
cd /cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/GR00T-WholeBodyControl/gear_sonic
|
| 151 |
+
export PYOPENGL_PLATFORM="egl"
|
| 152 |
+
export MUJOCO_GL="egl"
|
| 153 |
+
export QT_X11_NO_MITSHM=1
|
| 154 |
+
xvfb-run -a -s "-screen 0 1280x1024x24 -extension MIT-SHM" /cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/envs/gmr/bin/python scripts/batch_render_reference_motion_mujoco.py \
|
| 155 |
+
--motion-dir "$OUTPUT_DIR" \
|
| 156 |
+
--source-fps 50 \
|
| 157 |
+
--target-fps 30 \
|
| 158 |
+
--dof-order isaaclab
|
infer_render.sh
ADDED
|
@@ -0,0 +1,65 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
HFD_ROOT=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
|
| 2 |
+
source "$HFD_ROOT/scripts/activate_psi_env.sh"
|
| 3 |
+
cd /cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/Psi0-yth-v18
|
| 4 |
+
[ -f .env ] && source .env
|
| 5 |
+
|
| 6 |
+
export PYTHONPATH=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/Psi0-yth-v18/src
|
| 7 |
+
export CUDA_VISIBLE_DEVICES=0
|
| 8 |
+
export PSI_INFER_FLOAT32=1
|
| 9 |
+
pkill -f monitor.py
|
| 10 |
+
|
| 11 |
+
# <modify_audio_infer> 推理脚本切换为音频输入,对齐当前audio-only + full-audio + 39D TextOp训练逻辑。
|
| 12 |
+
# 原逻辑:
|
| 13 |
+
# RUN_DIR="/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/Psi0-yth-v18/runs/posttrain_v1/posttrain.sg.flow1000.cwml.lr1.0e-04.b2048.gpus8.2604250959"
|
| 14 |
+
# OUTPUT_DIR="/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/Psi0-yth-v18/runs/posttrain_v1/2604250959"
|
| 15 |
+
# python scripts/infer/seed_text_to_motion.py \
|
| 16 |
+
# --run-dir $RUN_DIR \
|
| 17 |
+
# --ckpt-step best \
|
| 18 |
+
# --output-dir $OUTPUT_DIR \
|
| 19 |
+
# --num-samples 1 \
|
| 20 |
+
# --start-index 23069 \
|
| 21 |
+
# --num-inference-steps 10 \
|
| 22 |
+
# --device cuda:0
|
| 23 |
+
|
| 24 |
+
RUN_TIMESTAMP=2604250426
|
| 25 |
+
RUN_PARENT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/Psi0-yth-v18/runs/posttrain_v1_audio_dance
|
| 26 |
+
RUN_DIR=$RUN_PARENT/posttrain.sag.flow1000.cwml.lr1.0e-04.b512.gpus8.$RUN_TIMESTAMP
|
| 27 |
+
OUTPUT_DIR=$RUN_PARENT/$RUN_TIMESTAMP.best
|
| 28 |
+
MOTION_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/GR00T-WholeBodyControl/gear_sonic_deploy/reference/converted/psi0_dance/gvhmr
|
| 29 |
+
VIDEO_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/dance_1723_video_psi0/dance_1723_video_10s
|
| 30 |
+
MOTION_NAME=movies_1mzl0Zh614A_0000
|
| 31 |
+
MOTION_DIR=$MOTION_ROOT/$MOTION_NAME
|
| 32 |
+
VIDEO_PATH=$VIDEO_ROOT/$MOTION_NAME.mp4
|
| 33 |
+
CKPT_STEP=best
|
| 34 |
+
PROMPT="Generate a dance sequence that matches the provided music."
|
| 35 |
+
|
| 36 |
+
rm -rf $OUTPUT_DIR
|
| 37 |
+
|
| 38 |
+
echo "Run dir: $RUN_DIR"
|
| 39 |
+
echo "Ckpt step: $CKPT_STEP"
|
| 40 |
+
echo "Prompt: $PROMPT"
|
| 41 |
+
echo "Motion dir: $MOTION_DIR"
|
| 42 |
+
echo "Video path: $VIDEO_PATH"
|
| 43 |
+
echo "Output dir: $OUTPUT_DIR"
|
| 44 |
+
|
| 45 |
+
python scripts/infer/seed_audio_to_motion.py \
|
| 46 |
+
--run-dir "$RUN_DIR" \
|
| 47 |
+
--ckpt-step "$CKPT_STEP" \
|
| 48 |
+
--output-dir "$OUTPUT_DIR" \
|
| 49 |
+
--motion-dir "$MOTION_DIR" \
|
| 50 |
+
--video-path "$VIDEO_PATH" \
|
| 51 |
+
--prompt "$PROMPT" \
|
| 52 |
+
--num-inference-steps 10 \
|
| 53 |
+
--device cuda:0
|
| 54 |
+
|
| 55 |
+
deactivate
|
| 56 |
+
unset PYTHONPATH
|
| 57 |
+
cd /cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/GR00T-WholeBodyControl/gear_sonic
|
| 58 |
+
conda activate /cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/envs/gmr
|
| 59 |
+
export PYOPENGL_PLATFORM="egl"
|
| 60 |
+
export MUJOCO_GL="egl"
|
| 61 |
+
xvfb-run -a python scripts/batch_render_reference_motion_mujoco.py \
|
| 62 |
+
--motion-dir "$OUTPUT_DIR" \
|
| 63 |
+
--source-fps 50 \
|
| 64 |
+
--target-fps 30 \
|
| 65 |
+
--dof-order isaaclab
|
infer_render_auto.sh
ADDED
|
@@ -0,0 +1,454 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#!/usr/bin/env bash
|
| 2 |
+
set -Eeuo pipefail
|
| 3 |
+
|
| 4 |
+
PROJECT_ROOT=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
|
| 5 |
+
cd "$PROJECT_ROOT"
|
| 6 |
+
export PYTHONPATH="$PROJECT_ROOT/src${PYTHONPATH:+:$PYTHONPATH}"
|
| 7 |
+
source "$PROJECT_ROOT/scripts/activate_psi_env.sh"
|
| 8 |
+
|
| 9 |
+
HFD_ENV_FILE=${HFD_ENV_FILE:-$PROJECT_ROOT/.env}
|
| 10 |
+
if [[ -f "$HFD_ENV_FILE" ]]; then
|
| 11 |
+
set -a
|
| 12 |
+
source "$HFD_ENV_FILE"
|
| 13 |
+
set +a
|
| 14 |
+
fi
|
| 15 |
+
|
| 16 |
+
PYTHON_BIN=${HFD_INFER_PYTHON:-python}
|
| 17 |
+
RUNS_ROOT=${HFD_INFER_RUNS_ROOT:-$PROJECT_ROOT/runs}
|
| 18 |
+
RUN_DIR=${HFD_INFER_RUN_DIR:-auto}
|
| 19 |
+
if [[ $# -gt 0 ]]; then
|
| 20 |
+
RUN_DIR=$1
|
| 21 |
+
fi
|
| 22 |
+
if [[ "$RUN_DIR" == "auto" ]]; then
|
| 23 |
+
RUN_DIR=$($PYTHON_BIN - "$RUNS_ROOT" <<'PY'
|
| 24 |
+
import json
|
| 25 |
+
import sys
|
| 26 |
+
from pathlib import Path
|
| 27 |
+
|
| 28 |
+
candidates = []
|
| 29 |
+
for config_path in Path(sys.argv[1]).rglob("run_config.json"):
|
| 30 |
+
try:
|
| 31 |
+
config = json.loads(config_path.read_text(encoding="utf-8"))
|
| 32 |
+
except Exception:
|
| 33 |
+
continue
|
| 34 |
+
model = config.get("model", {})
|
| 35 |
+
data = config.get("data", {})
|
| 36 |
+
train = config.get("train", {})
|
| 37 |
+
if (
|
| 38 |
+
not model.get("streaming_qwen_enabled", False)
|
| 39 |
+
or not data.get("online_audio", False)
|
| 40 |
+
or train.get("full_duplex_semantics_version") not in {9, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24}
|
| 41 |
+
):
|
| 42 |
+
continue
|
| 43 |
+
candidates.append((config_path.stat().st_mtime_ns, config_path.parent))
|
| 44 |
+
if candidates:
|
| 45 |
+
print(max(candidates, key=lambda item: item[0])[1])
|
| 46 |
+
PY
|
| 47 |
+
)
|
| 48 |
+
fi
|
| 49 |
+
if [[ -z "$RUN_DIR" || ! -f "$RUN_DIR/run_config.json" ]]; then
|
| 50 |
+
echo "[error] Cannot find a full-duplex RUN_DIR. Pass it as the first argument." >&2
|
| 51 |
+
exit 2
|
| 52 |
+
fi
|
| 53 |
+
RUN_DIR=$(cd "$RUN_DIR" && pwd)
|
| 54 |
+
|
| 55 |
+
if [[ -n "${HFD_INFER_GPUS:-}" ]]; then
|
| 56 |
+
export CUDA_VISIBLE_DEVICES=$HFD_INFER_GPUS
|
| 57 |
+
elif [[ -z "${CUDA_VISIBLE_DEVICES:-}" ]]; then
|
| 58 |
+
export CUDA_VISIBLE_DEVICES=0
|
| 59 |
+
fi
|
| 60 |
+
DEVICE=${HFD_INFER_DEVICE:-cuda:0}
|
| 61 |
+
CHECK_INTERVAL=${HFD_INFER_CHECK_INTERVAL:-30}
|
| 62 |
+
INFER_ONCE=${HFD_INFER_ONCE:-0}
|
| 63 |
+
MOTION_FRAMES=${HFD_INFER_MOTION_FRAMES:-450}
|
| 64 |
+
INFERENCE_STEPS=${HFD_INFER_STEPS:-10}
|
| 65 |
+
PROMPT=${HFD_INFER_PROMPT:-auto}
|
| 66 |
+
RESPECT_CONTROL=${HFD_INFER_RESPECT_CONTROL:-1}
|
| 67 |
+
MAX_LISTEN_TAIL_STEPS=${HFD_INFER_MAX_LISTEN_TAIL_STEPS:-500}
|
| 68 |
+
INITIAL_IDLE_STEPS=${HFD_INFER_INITIAL_IDLE_STEPS:-5}
|
| 69 |
+
WHITE_NOISE_START_INDEX=${HFD_INFER_WHITE_NOISE_START_INDEX:-3}
|
| 70 |
+
OUTPUT_SUBDIR=${HFD_INFER_OUTPUT_SUBDIR:-full_duplex_auto_infer}
|
| 71 |
+
OUTPUT_ROOT=${HFD_INFER_OUTPUT_ROOT:-$RUN_DIR/$OUTPUT_SUBDIR}
|
| 72 |
+
GMR_PY=${HFD_RENDER_PYTHON:-/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/envs/gmr/bin/python}
|
| 73 |
+
GMR_XML=${HFD_RENDER_XML:-/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/GMR/assets/unitree_g1/g1_mocap_29dof.xml}
|
| 74 |
+
SONIC_RENDER_ROOT=${HFD_RENDER_ROOT:-/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/GR00T-WholeBodyControl/gear_sonic}
|
| 75 |
+
RENDER_FPS=${HFD_RENDER_FPS:-50}
|
| 76 |
+
TERMINAL_BACKGROUND_STEPS=${HFD_INFER_TERMINAL_BACKGROUND_STEPS:-0}
|
| 77 |
+
INTER_TURN_BACKGROUND_STEPS_BY_TURN=${HFD_INFER_INTER_TURN_BACKGROUND_STEPS_BY_TURN:-}
|
| 78 |
+
MIN_CHECKPOINT_STEP=${HFD_INFER_MIN_CHECKPOINT_STEP:-0}
|
| 79 |
+
EXACT_CHECKPOINT_STEP=${HFD_INFER_EXACT_CHECKPOINT_STEP:-}
|
| 80 |
+
SAMPLE_TAG=${HFD_INFER_SAMPLE_TAG:-}
|
| 81 |
+
SAMPLE_PARENT=${HFD_INFER_SAMPLE_PARENT:-}
|
| 82 |
+
SCENARIO_FILE=${HFD_INFER_SCENARIO_FILE:-}
|
| 83 |
+
|
| 84 |
+
if [[ -n "$EXACT_CHECKPOINT_STEP" && ! "$EXACT_CHECKPOINT_STEP" =~ ^[0-9]+$ ]]; then
|
| 85 |
+
echo "[error] HFD_INFER_EXACT_CHECKPOINT_STEP must be a non-negative integer." >&2
|
| 86 |
+
exit 2
|
| 87 |
+
fi
|
| 88 |
+
if [[ -n "$SAMPLE_PARENT" && ! "$SAMPLE_PARENT" =~ ^[[:alnum:]_.-]+$ ]]; then
|
| 89 |
+
echo "[error] HFD_INFER_SAMPLE_PARENT must be one safe path component." >&2
|
| 90 |
+
exit 2
|
| 91 |
+
fi
|
| 92 |
+
|
| 93 |
+
# Default sample is one of the 50 fixed overfit sources (manifest row 48,
|
| 94 |
+
# "character walks forward while grabbing something"), using the exact fixed
|
| 95 |
+
# voice the trainer binds to that row (row_id 48 % 8 voices = female_01_en)
|
| 96 |
+
# and a verified neutral-idle motion as the initial pose. Starting directly
|
| 97 |
+
# from the task GT first frame is outside the training construction, which
|
| 98 |
+
# always begins from an audited upright stationary anchor and learns a 20-frame
|
| 99 |
+
# transition into the task. The previous default
|
| 100 |
+
# (003771, walk back-left) is not in the overfit set: a memorizing model has
|
| 101 |
+
# never seen a back-left walk and can only replay the forward walk it knows.
|
| 102 |
+
DEFAULT_AUDIO=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/qwen3-tts/seed_audio/000006/female_01_en.wav
|
| 103 |
+
DEFAULT_INIT_MOTION=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/Psi0/dataset/seed_all/g1/csv_sonic/221125/neutral_idle_loop_002__A073
|
| 104 |
+
AUDIO_PATHS_CSV=${HFD_INFER_AUDIO_PATHS:-$DEFAULT_AUDIO}
|
| 105 |
+
ADDITIONAL_AUDIO_PATHS_CSV=${HFD_INFER_ADDITIONAL_AUDIO_PATHS:-}
|
| 106 |
+
INTER_TURN_BACKGROUND_STEPS=${HFD_INFER_INTER_TURN_BACKGROUND_STEPS:-0}
|
| 107 |
+
INIT_MOTION_DIRS_CSV=${HFD_INFER_INIT_MOTION_DIRS:-$DEFAULT_INIT_MOTION}
|
| 108 |
+
IFS=',' read -r -a AUDIO_PATHS <<< "$AUDIO_PATHS_CSV"
|
| 109 |
+
ADDITIONAL_AUDIO_PATHS=()
|
| 110 |
+
if [[ -n "$ADDITIONAL_AUDIO_PATHS_CSV" ]]; then
|
| 111 |
+
IFS=',' read -r -a ADDITIONAL_AUDIO_PATHS <<< "$ADDITIONAL_AUDIO_PATHS_CSV"
|
| 112 |
+
fi
|
| 113 |
+
IFS=',' read -r -a INIT_MOTION_DIRS <<< "$INIT_MOTION_DIRS_CSV"
|
| 114 |
+
if [[ ${#INIT_MOTION_DIRS[@]} -ne 1 && ${#INIT_MOTION_DIRS[@]} -ne ${#AUDIO_PATHS[@]} ]]; then
|
| 115 |
+
echo "[error] HFD_INFER_INIT_MOTION_DIRS must contain one path or match HFD_INFER_AUDIO_PATHS." >&2
|
| 116 |
+
exit 2
|
| 117 |
+
fi
|
| 118 |
+
|
| 119 |
+
checkpoint_ready() {
|
| 120 |
+
local checkpoint_dir=$1
|
| 121 |
+
"$PYTHON_BIN" - "$checkpoint_dir" <<'PY'
|
| 122 |
+
import json
|
| 123 |
+
import sys
|
| 124 |
+
from pathlib import Path
|
| 125 |
+
|
| 126 |
+
checkpoint = Path(sys.argv[1])
|
| 127 |
+
single = checkpoint / "model.safetensors"
|
| 128 |
+
manifest = checkpoint / "hfd_fused_delta_manifest.json"
|
| 129 |
+
boundary = checkpoint / "full_duplex_source_boundary.json"
|
| 130 |
+
if (
|
| 131 |
+
single.is_file()
|
| 132 |
+
and single.stat().st_size > 0
|
| 133 |
+
and manifest.is_file()
|
| 134 |
+
and manifest.stat().st_size > 0
|
| 135 |
+
and boundary.is_file()
|
| 136 |
+
and boundary.stat().st_size > 0
|
| 137 |
+
):
|
| 138 |
+
payload = json.loads(boundary.read_text(encoding="utf-8"))
|
| 139 |
+
if bool(payload.get("source_group_complete", False)):
|
| 140 |
+
raise SystemExit(0)
|
| 141 |
+
raise SystemExit(1)
|
| 142 |
+
PY
|
| 143 |
+
}
|
| 144 |
+
|
| 145 |
+
checkpoint_step() {
|
| 146 |
+
local checkpoint_name=$1
|
| 147 |
+
if [[ "$checkpoint_name" == "best_ckpt" ]]; then
|
| 148 |
+
echo best
|
| 149 |
+
else
|
| 150 |
+
echo "${checkpoint_name#ckpt_}"
|
| 151 |
+
fi
|
| 152 |
+
}
|
| 153 |
+
|
| 154 |
+
checkpoint_output_name() {
|
| 155 |
+
local checkpoint_dir=$1
|
| 156 |
+
local captured_global_step=${2:-}
|
| 157 |
+
local checkpoint_name
|
| 158 |
+
checkpoint_name=$(basename "$checkpoint_dir")
|
| 159 |
+
if [[ "$checkpoint_name" != "best_ckpt" ]]; then
|
| 160 |
+
echo "$checkpoint_name"
|
| 161 |
+
return
|
| 162 |
+
fi
|
| 163 |
+
if [[ -z "$captured_global_step" ]]; then
|
| 164 |
+
captured_global_step=$(checkpoint_global_step "$checkpoint_dir")
|
| 165 |
+
fi
|
| 166 |
+
echo "best_ckpt_step_${captured_global_step}"
|
| 167 |
+
}
|
| 168 |
+
|
| 169 |
+
checkpoint_global_step() {
|
| 170 |
+
local checkpoint_dir=$1
|
| 171 |
+
"$PYTHON_BIN" - "$checkpoint_dir/full_duplex_source_boundary.json" <<'PY'
|
| 172 |
+
import json
|
| 173 |
+
import sys
|
| 174 |
+
from pathlib import Path
|
| 175 |
+
|
| 176 |
+
marker_path = Path(sys.argv[1])
|
| 177 |
+
if not marker_path.is_file():
|
| 178 |
+
raise SystemExit(f"checkpoint has no source-boundary marker: {marker_path}")
|
| 179 |
+
payload = json.loads(marker_path.read_text(encoding="utf-8"))
|
| 180 |
+
step = int(payload["global_step"])
|
| 181 |
+
if step <= 0:
|
| 182 |
+
raise SystemExit(f"checkpoint has invalid global_step={step}")
|
| 183 |
+
print(step)
|
| 184 |
+
PY
|
| 185 |
+
}
|
| 186 |
+
|
| 187 |
+
safe_sample_name() {
|
| 188 |
+
local audio_path=$1
|
| 189 |
+
local index=$2
|
| 190 |
+
local stem
|
| 191 |
+
stem=$(basename "$audio_path")
|
| 192 |
+
stem=${stem%.*}
|
| 193 |
+
stem=$(echo "$stem" | tr -cs '[:alnum:]_.-' '_')
|
| 194 |
+
if [[ -n "$SAMPLE_TAG" ]]; then
|
| 195 |
+
stem="${SAMPLE_TAG}_${stem}"
|
| 196 |
+
fi
|
| 197 |
+
if [[ ${#ADDITIONAL_AUDIO_PATHS[@]} -gt 0 ]]; then
|
| 198 |
+
printf '%02d_%s_session%dturn' \
|
| 199 |
+
"$index" "$stem" "$(( ${#ADDITIONAL_AUDIO_PATHS[@]} + 1 ))"
|
| 200 |
+
else
|
| 201 |
+
printf '%02d_%s' "$index" "$stem"
|
| 202 |
+
fi
|
| 203 |
+
}
|
| 204 |
+
|
| 205 |
+
sample_output_dir() {
|
| 206 |
+
local checkpoint_name=$1
|
| 207 |
+
local sample_name=$2
|
| 208 |
+
if [[ -n "$SAMPLE_PARENT" ]]; then
|
| 209 |
+
printf '%s/%s/%s/%s\n' \
|
| 210 |
+
"$OUTPUT_ROOT" "$checkpoint_name" "$SAMPLE_PARENT" "$sample_name"
|
| 211 |
+
else
|
| 212 |
+
printf '%s/%s/%s\n' "$OUTPUT_ROOT" "$checkpoint_name" "$sample_name"
|
| 213 |
+
fi
|
| 214 |
+
}
|
| 215 |
+
|
| 216 |
+
render_sample() {
|
| 217 |
+
local sample_dir=$1
|
| 218 |
+
(
|
| 219 |
+
cd "$SONIC_RENDER_ROOT"
|
| 220 |
+
env -u PYTHONPATH \
|
| 221 |
+
PYOPENGL_PLATFORM=egl \
|
| 222 |
+
MUJOCO_GL=egl \
|
| 223 |
+
QT_X11_NO_MITSHM=1 \
|
| 224 |
+
SONIC_RENDER_HEADLESS_CAMERA=1 \
|
| 225 |
+
SONIC_RENDER_LOCAL_TMP_OUTPUT=1 \
|
| 226 |
+
SONIC_RENDER_TMPDIR=/tmp \
|
| 227 |
+
"$GMR_PY" scripts/batch_render_reference_motion_mujoco.py \
|
| 228 |
+
--motion-dir "$sample_dir" \
|
| 229 |
+
--xml-path "$GMR_XML" \
|
| 230 |
+
--source-fps 50 \
|
| 231 |
+
--target-fps "$RENDER_FPS" \
|
| 232 |
+
--dof-order isaaclab
|
| 233 |
+
)
|
| 234 |
+
}
|
| 235 |
+
|
| 236 |
+
process_sample() {
|
| 237 |
+
local checkpoint_name=$1
|
| 238 |
+
local step=$2
|
| 239 |
+
local sample_index=$3
|
| 240 |
+
local expected_global_step=$4
|
| 241 |
+
local audio_path=${AUDIO_PATHS[$sample_index]}
|
| 242 |
+
local init_index=$sample_index
|
| 243 |
+
if [[ ${#INIT_MOTION_DIRS[@]} -eq 1 ]]; then
|
| 244 |
+
init_index=0
|
| 245 |
+
fi
|
| 246 |
+
local init_motion_dir=${INIT_MOTION_DIRS[$init_index]}
|
| 247 |
+
local sample_name
|
| 248 |
+
sample_name=$(safe_sample_name "$audio_path" "$sample_index")
|
| 249 |
+
local sample_dir
|
| 250 |
+
sample_dir=$(sample_output_dir "$checkpoint_name" "$sample_name")
|
| 251 |
+
mkdir -p "$sample_dir"
|
| 252 |
+
|
| 253 |
+
if [[ ! -s "$sample_dir/infer_meta.json" || ! -s "$sample_dir/body_pos.csv" ]]; then
|
| 254 |
+
echo "[infer] checkpoint=$checkpoint_name sample=$sample_name audio=$audio_path"
|
| 255 |
+
control_args=()
|
| 256 |
+
additional_audio_args=()
|
| 257 |
+
inter_turn_args=()
|
| 258 |
+
for additional_audio_path in "${ADDITIONAL_AUDIO_PATHS[@]}"; do
|
| 259 |
+
additional_audio_args+=(--additional-audio-path "$additional_audio_path")
|
| 260 |
+
done
|
| 261 |
+
if [[ -n "$INTER_TURN_BACKGROUND_STEPS_BY_TURN" ]]; then
|
| 262 |
+
inter_turn_args+=(
|
| 263 |
+
--inter-turn-background-steps-by-turn
|
| 264 |
+
"$INTER_TURN_BACKGROUND_STEPS_BY_TURN"
|
| 265 |
+
)
|
| 266 |
+
fi
|
| 267 |
+
if [[ "$RESPECT_CONTROL" != "1" && "$RESPECT_CONTROL" != "true" ]]; then
|
| 268 |
+
control_args+=(--ignore-control-token)
|
| 269 |
+
fi
|
| 270 |
+
rm -f "$sample_dir/infer_failure.json"
|
| 271 |
+
if "$PYTHON_BIN" -u scripts/infer/full_duplex_audio_to_motion.py \
|
| 272 |
+
--run-dir "$RUN_DIR" \
|
| 273 |
+
--ckpt-step "$step" \
|
| 274 |
+
--output-dir "$sample_dir" \
|
| 275 |
+
--audio-path "$audio_path" \
|
| 276 |
+
"${additional_audio_args[@]}" \
|
| 277 |
+
--init-motion-dir "$init_motion_dir" \
|
| 278 |
+
--motion-frames "$MOTION_FRAMES" \
|
| 279 |
+
--inter-turn-background-steps "$INTER_TURN_BACKGROUND_STEPS" \
|
| 280 |
+
"${inter_turn_args[@]}" \
|
| 281 |
+
--terminal-background-steps "$TERMINAL_BACKGROUND_STEPS" \
|
| 282 |
+
--num-inference-steps "$INFERENCE_STEPS" \
|
| 283 |
+
--prompt "$PROMPT" \
|
| 284 |
+
--max-listen-tail-steps "$MAX_LISTEN_TAIL_STEPS" \
|
| 285 |
+
--initial-idle-steps "$INITIAL_IDLE_STEPS" \
|
| 286 |
+
--white-noise-start-index "$WHITE_NOISE_START_INDEX" \
|
| 287 |
+
--expected-checkpoint-global-step "$expected_global_step" \
|
| 288 |
+
--device "$DEVICE" \
|
| 289 |
+
"${control_args[@]}" 2>&1 | tee "$sample_dir/infer.log"; then
|
| 290 |
+
:
|
| 291 |
+
else
|
| 292 |
+
local infer_status=${PIPESTATUS[0]}
|
| 293 |
+
return "$infer_status"
|
| 294 |
+
fi
|
| 295 |
+
else
|
| 296 |
+
echo "[skip] inference exists: $sample_dir"
|
| 297 |
+
fi
|
| 298 |
+
|
| 299 |
+
if [[ ! -s "$sample_dir/reference_motion.mp4" ]]; then
|
| 300 |
+
echo "[render] checkpoint=$checkpoint_name sample=$sample_name fps=$RENDER_FPS"
|
| 301 |
+
if ! render_sample "$sample_dir"; then
|
| 302 |
+
echo "[error] render failed: $sample_dir" >&2
|
| 303 |
+
return 1
|
| 304 |
+
fi
|
| 305 |
+
else
|
| 306 |
+
echo "[skip] render exists: $sample_dir/reference_motion.mp4"
|
| 307 |
+
fi
|
| 308 |
+
|
| 309 |
+
if [[ ! -s "$sample_dir/reference_motion_with_audio.mp4" ]]; then
|
| 310 |
+
echo "[mux] use the complete background/speech/background streaming audio timeline"
|
| 311 |
+
if ! "$PYTHON_BIN" scripts/infer/mux_full_duplex_audio.py \
|
| 312 |
+
--video "$sample_dir/reference_motion.mp4" \
|
| 313 |
+
--audio "$sample_dir/assembled_input_audio_16k.wav" \
|
| 314 |
+
--output "$sample_dir/reference_motion_with_audio.mp4" \
|
| 315 |
+
--infer-meta "$sample_dir/infer_meta.json" \
|
| 316 |
+
--audio-is-timeline \
|
| 317 |
+
--expected-fps "$RENDER_FPS"; then
|
| 318 |
+
echo "[error] audio mux failed: $sample_dir" >&2
|
| 319 |
+
return 1
|
| 320 |
+
fi
|
| 321 |
+
else
|
| 322 |
+
echo "[skip] synchronized video exists: $sample_dir/reference_motion_with_audio.mp4"
|
| 323 |
+
fi
|
| 324 |
+
touch "$sample_dir/.complete"
|
| 325 |
+
}
|
| 326 |
+
|
| 327 |
+
mkdir -p "$OUTPUT_ROOT"
|
| 328 |
+
if [[ -n "$SCENARIO_FILE" ]]; then
|
| 329 |
+
if [[ ! -f "$SCENARIO_FILE" ]]; then
|
| 330 |
+
echo "[error] HFD_INFER_SCENARIO_FILE does not exist: $SCENARIO_FILE" >&2
|
| 331 |
+
exit 2
|
| 332 |
+
fi
|
| 333 |
+
scenario_snapshot="$OUTPUT_ROOT/scenario.snapshot.json"
|
| 334 |
+
if [[ -n "$SAMPLE_PARENT" ]]; then
|
| 335 |
+
scenario_snapshot="$OUTPUT_ROOT/scenario.${SAMPLE_PARENT}.snapshot.json"
|
| 336 |
+
fi
|
| 337 |
+
cp -- "$SCENARIO_FILE" "$scenario_snapshot"
|
| 338 |
+
fi
|
| 339 |
+
if [[ "$MIN_CHECKPOINT_STEP" == "auto" ]]; then
|
| 340 |
+
MIN_CHECKPOINT_STEP=0
|
| 341 |
+
if [[ -z "$EXACT_CHECKPOINT_STEP" ]]; then
|
| 342 |
+
while IFS= read -r candidate_checkpoint; do
|
| 343 |
+
checkpoint_ready "$candidate_checkpoint" || continue
|
| 344 |
+
candidate_name=$(basename "$candidate_checkpoint")
|
| 345 |
+
candidate_step=${candidate_name#ckpt_}
|
| 346 |
+
if (( candidate_step > MIN_CHECKPOINT_STEP )); then
|
| 347 |
+
MIN_CHECKPOINT_STEP=$candidate_step
|
| 348 |
+
fi
|
| 349 |
+
done < <(
|
| 350 |
+
find "$RUN_DIR/checkpoints" -mindepth 1 -maxdepth 1 -type d \
|
| 351 |
+
-name 'ckpt_[0-9]*' -print 2>/dev/null
|
| 352 |
+
)
|
| 353 |
+
fi
|
| 354 |
+
fi
|
| 355 |
+
if [[ ! "$MIN_CHECKPOINT_STEP" =~ ^[0-9]+$ ]]; then
|
| 356 |
+
echo "[error] HFD_INFER_MIN_CHECKPOINT_STEP must be auto or a non-negative integer." >&2
|
| 357 |
+
exit 2
|
| 358 |
+
fi
|
| 359 |
+
echo "RUN_DIR: $RUN_DIR"
|
| 360 |
+
echo "OUTPUT_ROOT: $OUTPUT_ROOT"
|
| 361 |
+
echo "TIMING: audio 16kHz, 0.6s window, 0.4s stride; action 30/20/10 at 50fps"
|
| 362 |
+
echo "RENDER: ${RENDER_FPS}fps; speech follows ${INITIAL_IDLE_STEPS} idle windows; motion begins at recorded frame boundary"
|
| 363 |
+
echo "CONTROL: respect=$RESPECT_CONTROL initial_idle_steps=$INITIAL_IDLE_STEPS max_listen_tail_steps=$MAX_LISTEN_TAIL_STEPS white_noise_start=$WHITE_NOISE_START_INDEX"
|
| 364 |
+
echo "SESSION: primary_samples=${#AUDIO_PATHS[@]} additional_turns=${#ADDITIONAL_AUDIO_PATHS[@]} inter_turn_background_steps=$INTER_TURN_BACKGROUND_STEPS by_turn=${INTER_TURN_BACKGROUND_STEPS_BY_TURN:-uniform} terminal_background_steps=$TERMINAL_BACKGROUND_STEPS"
|
| 365 |
+
echo "CHECKPOINT POLICY: immutable ckpt_<step> only; mutable best_ckpt is never watched; min_step=$MIN_CHECKPOINT_STEP exact_step=${EXACT_CHECKPOINT_STEP:-none}"
|
| 366 |
+
|
| 367 |
+
while true; do
|
| 368 |
+
if [[ -n "$EXACT_CHECKPOINT_STEP" ]]; then
|
| 369 |
+
CHECKPOINT_DIRS=("$RUN_DIR/checkpoints/ckpt_$EXACT_CHECKPOINT_STEP")
|
| 370 |
+
else
|
| 371 |
+
mapfile -t CHECKPOINT_DIRS < <(
|
| 372 |
+
find "$RUN_DIR/checkpoints" -mindepth 1 -maxdepth 1 -type d \
|
| 373 |
+
-name 'ckpt_[0-9]*' -print 2>/dev/null | sort -V
|
| 374 |
+
)
|
| 375 |
+
fi
|
| 376 |
+
pending=()
|
| 377 |
+
for checkpoint_dir in "${CHECKPOINT_DIRS[@]}"; do
|
| 378 |
+
[[ -d "$checkpoint_dir" ]] || continue
|
| 379 |
+
checkpoint_dir_name=$(basename "$checkpoint_dir")
|
| 380 |
+
checkpoint_number=${checkpoint_dir_name#ckpt_}
|
| 381 |
+
if (( checkpoint_number < MIN_CHECKPOINT_STEP )); then
|
| 382 |
+
continue
|
| 383 |
+
fi
|
| 384 |
+
checkpoint_ready "$checkpoint_dir" || continue
|
| 385 |
+
if ! captured_global_step=$(checkpoint_global_step "$checkpoint_dir"); then
|
| 386 |
+
echo "[wait] checkpoint identity is not ready: $checkpoint_dir" >&2
|
| 387 |
+
continue
|
| 388 |
+
fi
|
| 389 |
+
checkpoint_name=$(checkpoint_output_name "$checkpoint_dir" "$captured_global_step")
|
| 390 |
+
settled=1
|
| 391 |
+
for sample_index in "${!AUDIO_PATHS[@]}"; do
|
| 392 |
+
sample_name=$(safe_sample_name "${AUDIO_PATHS[$sample_index]}" "$sample_index")
|
| 393 |
+
sample_dir=$(sample_output_dir "$checkpoint_name" "$sample_name")
|
| 394 |
+
if [[ ! -f "$sample_dir/.complete" && ! -f "$sample_dir/.failed" ]]; then
|
| 395 |
+
settled=0
|
| 396 |
+
break
|
| 397 |
+
fi
|
| 398 |
+
done
|
| 399 |
+
[[ $settled -eq 1 ]] || pending+=("$checkpoint_dir")
|
| 400 |
+
done
|
| 401 |
+
|
| 402 |
+
if [[ ${#pending[@]} -eq 0 ]]; then
|
| 403 |
+
echo "[$(date '+%F %T')] no pending checkpoint"
|
| 404 |
+
if [[ "$INFER_ONCE" == "1" || "$INFER_ONCE" == "true" ]]; then
|
| 405 |
+
break
|
| 406 |
+
fi
|
| 407 |
+
sleep "$CHECK_INTERVAL"
|
| 408 |
+
continue
|
| 409 |
+
fi
|
| 410 |
+
|
| 411 |
+
checkpoint_dir=${pending[-1]}
|
| 412 |
+
checkpoint_dir_name=$(basename "$checkpoint_dir")
|
| 413 |
+
if ! expected_global_step=$(checkpoint_global_step "$checkpoint_dir"); then
|
| 414 |
+
echo "[wait] selected checkpoint identity changed before inference: $checkpoint_dir" >&2
|
| 415 |
+
continue
|
| 416 |
+
fi
|
| 417 |
+
checkpoint_name=$(checkpoint_output_name "$checkpoint_dir" "$expected_global_step")
|
| 418 |
+
step=$(checkpoint_step "$checkpoint_dir_name")
|
| 419 |
+
echo "[$(date '+%F %T')] process newest pending checkpoint: $checkpoint_name"
|
| 420 |
+
for sample_index in "${!AUDIO_PATHS[@]}"; do
|
| 421 |
+
sample_name=$(safe_sample_name "${AUDIO_PATHS[$sample_index]}" "$sample_index")
|
| 422 |
+
sample_dir=$(sample_output_dir "$checkpoint_name" "$sample_name")
|
| 423 |
+
if [[ -f "$sample_dir/.complete" ]]; then
|
| 424 |
+
echo "[skip] completed sample: $sample_dir"
|
| 425 |
+
continue
|
| 426 |
+
fi
|
| 427 |
+
if [[ -f "$sample_dir/.failed" ]]; then
|
| 428 |
+
echo "[skip] failed sample already recorded: $sample_dir"
|
| 429 |
+
continue
|
| 430 |
+
fi
|
| 431 |
+
if process_sample \
|
| 432 |
+
"$checkpoint_name" \
|
| 433 |
+
"$step" \
|
| 434 |
+
"$sample_index" \
|
| 435 |
+
"$expected_global_step"; then
|
| 436 |
+
continue
|
| 437 |
+
else
|
| 438 |
+
status=$?
|
| 439 |
+
printf \
|
| 440 |
+
'checkpoint=%s\nsample=%s\nexit_status=%s\nfailed_at=%s\n' \
|
| 441 |
+
"$checkpoint_name" \
|
| 442 |
+
"$sample_name" \
|
| 443 |
+
"$status" \
|
| 444 |
+
"$(date --iso-8601=seconds)" \
|
| 445 |
+
> "$sample_dir/.failed"
|
| 446 |
+
echo "[failed] checkpoint=$checkpoint_name sample=$sample_name status=$status; details: $sample_dir/infer.log" >&2
|
| 447 |
+
fi
|
| 448 |
+
done
|
| 449 |
+
if [[ "$INFER_ONCE" == "1" || "$INFER_ONCE" == "true" ]]; then
|
| 450 |
+
break
|
| 451 |
+
fi
|
| 452 |
+
done
|
| 453 |
+
|
| 454 |
+
echo "[done] automatic full-duplex inference/render loop exited cleanly"
|
plan_0.md
ADDED
|
@@ -0,0 +1,374 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Psi0 改造成 Text-to-Humanoid-Motion 计划(最小改动版)
|
| 2 |
+
|
| 3 |
+
## 0) 目标与约束
|
| 4 |
+
|
| 5 |
+
你的目标可以抽象成:
|
| 6 |
+
|
| 7 |
+
1. 输入:文本 caption(先不使用图像/音频)。
|
| 8 |
+
2. 输出:机器人动作序列(你定义为每帧 36 维,`29 dof + xyz + wxyz`)。
|
| 9 |
+
3. 训练策略:不走 pretrain,直接用 posttrain 框架,冻结 VLM,只训练 action expert(扩散动作头)。
|
| 10 |
+
4. 未来扩展:VLM 换到 `Qwen-2.5-Omni-3B`,当前先做 text-only smoke test。
|
| 11 |
+
5. 工程要求:尽可能少改现有代码。
|
| 12 |
+
|
| 13 |
+
下面的方案按这个目标严格展开。
|
| 14 |
+
|
| 15 |
+
---
|
| 16 |
+
|
| 17 |
+
## 1) 建议先看的代码(阅读顺序)
|
| 18 |
+
|
| 19 |
+
按这个顺序看,最快建立全局心智模型:
|
| 20 |
+
|
| 21 |
+
1. 训练主入口与配置注入
|
| 22 |
+
- `scripts/train.py`(训练循环、配置解析入口)
|
| 23 |
+
- `src/psi/config/config.py`(`LaunchConfig/TrainConfig/DataConfig/ModelConfig`)
|
| 24 |
+
- `src/psi/config/train/posttrain_he_psi0_config.py`
|
| 25 |
+
- `src/psi/config/train/posttrain_mix_psi0_config.py`
|
| 26 |
+
- `scripts/train/psi0/posttrain-he-psi0.sh`
|
| 27 |
+
|
| 28 |
+
2. posttrain trainer 主流程
|
| 29 |
+
- `src/psi/trainers/posttrain.py`
|
| 30 |
+
- `src/psi/trainers/trainer.py`
|
| 31 |
+
|
| 32 |
+
3. 数据层(raw -> transform -> batch)
|
| 33 |
+
- `src/psi/config/data_he.py`
|
| 34 |
+
- `src/psi/data/humanoid/he_raw_dataset.py`
|
| 35 |
+
- `src/psi/data/dataset.py`
|
| 36 |
+
- `src/psi/config/transform.py`(重点:`HEPosttrainRepackTransform`, `ActionStateTransform`, `Psi0ModelTransform`)
|
| 37 |
+
- `src/psi/trainers/qwen3vl_mixin.py`(`PaddedCollatorForTogether`)
|
| 38 |
+
|
| 39 |
+
4. 模型层(VLM + action expert)
|
| 40 |
+
- `src/psi/config/model_psi0.py`
|
| 41 |
+
- `src/psi/models/psi0.py`(重点:`Psi0Model.forward`, `ActionTransformerModel`, `ObservationProjection`)
|
| 42 |
+
|
| 43 |
+
5. 数据预处理脚本(参考)
|
| 44 |
+
- `scripts/data/raw_he_to_psi0.py`
|
| 45 |
+
- `scripts/data/raw_to_lerobot.py`
|
| 46 |
+
- `scripts/data/calc_modality_stats.py`
|
| 47 |
+
- `scripts/data/raw_dexmate_psi0.py`
|
| 48 |
+
|
| 49 |
+
---
|
| 50 |
+
|
| 51 |
+
## 2) 当前 posttrain 数据流和训练流(从数据处理开始)
|
| 52 |
+
|
| 53 |
+
## 2.1 训练入口与实例化
|
| 54 |
+
|
| 55 |
+
1. `scripts/train.py` 根据命令行第一个参数动态导入 config 模块(例如 `posttrain_he_psi0_config`)。
|
| 56 |
+
2. `Trainer.instantiate` 会按 `cfg.train.name` 选择 `psi.trainers.posttrain.PosttrainTrainer`。
|
| 57 |
+
3. 流程是固定的:`init_models -> create_datasets -> create_dataloaders -> create_optimizer_and_scheduler -> prepare -> training loop`。
|
| 58 |
+
|
| 59 |
+
对应代码:
|
| 60 |
+
- `scripts/train.py:169-219, 233-314, 350-360`
|
| 61 |
+
|
| 62 |
+
## 2.2 数据集原始样本(HE 例子)
|
| 63 |
+
|
| 64 |
+
`HERawDataset.__getitem__` 输出 raw dict,核心字段:
|
| 65 |
+
|
| 66 |
+
1. `observation.images.egocentric`: `(To, 3, H, W)`,默认 `To=1`。
|
| 67 |
+
2. `observation.hand_joints`, `observation.arm_joints`。
|
| 68 |
+
3. 展平后的 action 字段(例如 `action.joint_angles`)。
|
| 69 |
+
4. `task`(instruction 文本),`action_mask`,`obs_mask`。
|
| 70 |
+
|
| 71 |
+
对应代码:
|
| 72 |
+
- `src/psi/data/humanoid/he_raw_dataset.py:176-237`
|
| 73 |
+
|
| 74 |
+
## 2.3 三段 transform:repack -> field -> model
|
| 75 |
+
|
| 76 |
+
`MapStyleDataset.__getitem__` 会执行:
|
| 77 |
+
|
| 78 |
+
1. `repack`:把 raw 字段整理成统一结构。
|
| 79 |
+
2. `field`:做动作/状态归一化、padding、反归一化逻辑。
|
| 80 |
+
3. `model`:把图像+文本喂给 processor,生成 `input_ids/pixel_values/...`。
|
| 81 |
+
|
| 82 |
+
对应代码:
|
| 83 |
+
- `src/psi/data/dataset.py:23-26`
|
| 84 |
+
- `src/psi/config/transform.py`(`DataTransform.__call__`)
|
| 85 |
+
|
| 86 |
+
### repack(posttrain HE)
|
| 87 |
+
|
| 88 |
+
`HEPosttrainRepackTransform` 做的事:
|
| 89 |
+
|
| 90 |
+
1. 构造 `states`、`actions`。
|
| 91 |
+
2. 如果 `use_delta_actions=True`,将动作转成 delta(`actions[1:] - actions[:-1]`)。
|
| 92 |
+
3. 按 `pad_action_dim/pad_state_dim` 补齐。
|
| 93 |
+
4. 输出统一字段:`observations`, `states`, `actions`, `instruction`, `actions_mask` 等。
|
| 94 |
+
|
| 95 |
+
对应代码:
|
| 96 |
+
- `src/psi/config/transform.py:705-767`
|
| 97 |
+
|
| 98 |
+
### field(动作归一化)
|
| 99 |
+
|
| 100 |
+
`ActionStateTransform`:
|
| 101 |
+
|
| 102 |
+
1. 从 `stat_path` 读 `min/max` 或 `q01/q99`。
|
| 103 |
+
2. 对 `actions` 归一化到 `[-1,1]`。
|
| 104 |
+
3. 保留 `raw_actions`,给评估和反归一化使用。
|
| 105 |
+
|
| 106 |
+
对应代码:
|
| 107 |
+
- `src/psi/config/transform.py:408-578`
|
| 108 |
+
|
| 109 |
+
### model(构造 VLM 输入)
|
| 110 |
+
|
| 111 |
+
`Psi0ModelTransform`:
|
| 112 |
+
|
| 113 |
+
1. 图像 resize/crop/aug。
|
| 114 |
+
2. 构造 chat message:`[images] + text instruction`。
|
| 115 |
+
3. `vlm_processor(...)` 得到 `input_ids`, `attention_mask`, `pixel_values`, `image_grid_thw`。
|
| 116 |
+
4. 附带 `actions/states/actions_mask/raw_actions` 回传。
|
| 117 |
+
|
| 118 |
+
对应代码:
|
| 119 |
+
- `src/psi/config/transform.py:797-888`
|
| 120 |
+
|
| 121 |
+
## 2.4 collator 到 batch
|
| 122 |
+
|
| 123 |
+
`PaddedCollatorForTogether` 负责:
|
| 124 |
+
|
| 125 |
+
1. 对 `input_ids` padding。
|
| 126 |
+
2. 组装 `attention_mask`。
|
| 127 |
+
3. 强制 stack `pixel_values` 和 `image_grid_thw`。
|
| 128 |
+
4. stack `actions/states/actions_mask`。
|
| 129 |
+
|
| 130 |
+
这意味着目前代码路径默认“有视觉输入字段”。
|
| 131 |
+
|
| 132 |
+
对应代码:
|
| 133 |
+
- `src/psi/trainers/qwen3vl_mixin.py:87-157`
|
| 134 |
+
|
| 135 |
+
## 2.5 模型 forward(posttrain)
|
| 136 |
+
|
| 137 |
+
`PosttrainTrainer.forward_and_loss` 里调用:
|
| 138 |
+
|
| 139 |
+
1. 从 GT 动作加噪(flow 或 ddpm)。
|
| 140 |
+
2. `Psi0Model.forward(...)` 输入:
|
| 141 |
+
- `input_ids`, `attention_mask`, `pixel_values`, `image_grid_thw`
|
| 142 |
+
- `action_samples`(噪声动作)
|
| 143 |
+
- `states`
|
| 144 |
+
- `timestep`
|
| 145 |
+
3. `Psi0Model` ��跑 VLM 拿最后一层 hidden states,再喂 action expert(扩散动作头)。
|
| 146 |
+
4. 输出 `action_pred`,与目标噪声/速度算 MSE。
|
| 147 |
+
|
| 148 |
+
对应代码:
|
| 149 |
+
- `src/psi/trainers/posttrain.py:525-584`
|
| 150 |
+
- `src/psi/models/psi0.py:1602-1639`
|
| 151 |
+
|
| 152 |
+
## 2.6 训练损失与评估
|
| 153 |
+
|
| 154 |
+
1. loss 是动作维度 MSE,加 `actions_mask` 后按维度加权汇总。
|
| 155 |
+
2. `action_dim != 7` 时,默认均匀权重(你的 36 维会走这个分支)。
|
| 156 |
+
3. 评估里目前有固定分组切分 `[7,14,21]`,这是历史结构假设,对你的 36 维语义不一定成立。
|
| 157 |
+
|
| 158 |
+
对应代码:
|
| 159 |
+
- `src/psi/trainers/posttrain.py:55-64`
|
| 160 |
+
- `src/psi/trainers/posttrain.py:505-517`
|
| 161 |
+
|
| 162 |
+
---
|
| 163 |
+
|
| 164 |
+
## 3) 输入输出维度分别在哪里定义
|
| 165 |
+
|
| 166 |
+
### 3.1 动作/状态主维度
|
| 167 |
+
|
| 168 |
+
1. `--model.action-dim`:动作维度(你要设 36)。
|
| 169 |
+
2. `--model.odim`:状态维度(要和最终 `states` 最后一维一致)。
|
| 170 |
+
3. `--model.action-chunk-size`:每次预测 horizon(`Tp`)。
|
| 171 |
+
4. `--data.transform.repack.pad-action-dim`:repack 后动作 pad 到多少维。
|
| 172 |
+
5. `--data.transform.repack.pad-state-dim`:repack 后状态 pad 到多少维。
|
| 173 |
+
6. `--data.transform.field.pad-action-dim`:归一化统计也按这个维度对齐。
|
| 174 |
+
7. `--data.transform.field.pad-state-dim`:同上。
|
| 175 |
+
|
| 176 |
+
参考:
|
| 177 |
+
- `scripts/train/psi0/posttrain-he-psi0.sh:55-75`
|
| 178 |
+
- `src/psi/config/model_psi0.py:17-37`
|
| 179 |
+
- `src/psi/models/psi0.py:592-594, 757-761, 788, 1499-1503`
|
| 180 |
+
|
| 181 |
+
### 3.2 VLM hidden size 与 view_feature_dim
|
| 182 |
+
|
| 183 |
+
1. `ObservationProjection.views_proj = Linear(view_feature_dim -> hidden_dim)`。
|
| 184 |
+
2. 这里的 `view_feature_dim` 必须等于 VLM 最后一层 hidden size。
|
| 185 |
+
3. 当前脚本用 `--model.view_feature_dim=2048`(对应现有 Qwen3-VL-2B 配置)。
|
| 186 |
+
|
| 187 |
+
参考:
|
| 188 |
+
- `src/psi/models/psi0.py:499`
|
| 189 |
+
- `src/psi/models/psi0.py:1624, 1632`
|
| 190 |
+
- `scripts/train/psi0/posttrain-he-psi0.sh:75`
|
| 191 |
+
|
| 192 |
+
### 3.3 典型 batch 形状(当前 posttrain he 脚本)
|
| 193 |
+
|
| 194 |
+
1. `actions`: `(B, 16, 36)`
|
| 195 |
+
2. `states`: `(B, 1, 36)`(`observation_horizon=1`)
|
| 196 |
+
3. `input_ids`: `(B, L)`
|
| 197 |
+
4. `attention_mask`: `(B, L)`
|
| 198 |
+
5. `pixel_values`, `image_grid_thw`: 由 processor 决定
|
| 199 |
+
|
| 200 |
+
---
|
| 201 |
+
|
| 202 |
+
## 4) 面向你目标的最小改造方案
|
| 203 |
+
|
| 204 |
+
下面给两条路径,我建议先 A 再 B。
|
| 205 |
+
|
| 206 |
+
## 路径 A(最小改动,优先跑通)
|
| 207 |
+
|
| 208 |
+
核心思想:先保留现有视觉字段接口,不做“真正 text-only 张量路径”;数据侧喂一个固定占位图(dummy image),让模型主要学 text->motion。这样能最大复用 `Psi0ModelTransform + Collator + Psi0Model.forward`,代码改动最小。
|
| 209 |
+
|
| 210 |
+
你需要做:
|
| 211 |
+
|
| 212 |
+
1. 新增 `BoneSeedDataset`(或在现有 dataset 适配)输出统一字段:
|
| 213 |
+
- `observations`: 放一个固定 PIL 图(例如纯黑 240x320)
|
| 214 |
+
- `states`: `(To, 36)`,通常 `To=1`
|
| 215 |
+
- `actions`: `(Tp, 36)`
|
| 216 |
+
- `instruction`: caption
|
| 217 |
+
- `actions_mask`: `(Tp, 36)`
|
| 218 |
+
|
| 219 |
+
2. 新增一个 `DataConfig`(例如 `data_boneseed.py`)和一个训练 config(例如 `posttrain_boneseed_psi0_config.py`):
|
| 220 |
+
- `transform.repack`: 你的 `BoneSeedRepackTransform`
|
| 221 |
+
- `transform.field`: 复用 `ActionStateTransform`
|
| 222 |
+
- `transform.model`: 先复用 `Psi0ModelTransform`
|
| 223 |
+
|
| 224 |
+
3. 新增训练脚本(仿 `posttrain-he-psi0.sh`):
|
| 225 |
+
- 维度全设 36
|
| 226 |
+
- 先设置 `n_conditions=0`
|
| 227 |
+
- `tune_vlm=False`
|
| 228 |
+
- `noise_scheduler=flow`
|
| 229 |
+
|
| 230 |
+
优点:
|
| 231 |
+
|
| 232 |
+
1. 几乎不动模型与 trainer 主干。
|
| 233 |
+
2. 能最快 smoke test。
|
| 234 |
+
|
| 235 |
+
代价:
|
| 236 |
+
|
| 237 |
+
1. 严格来说不是“纯 text-only”实现,只是视觉分支恒定输入。
|
| 238 |
+
|
| 239 |
+
## 路径 B(真正 text-only)
|
| 240 |
+
|
| 241 |
+
需要改 3 个点:
|
| 242 |
+
|
| 243 |
+
1. `Psi0ModelTransform.build_qwenvl_inputs`:
|
| 244 |
+
- 允许 `imgs=[]`。
|
| 245 |
+
- `process_vision_info` 返回 `None` 时,仍可仅返回 text tokens。
|
| 246 |
+
|
| 247 |
+
2. `PaddedCollatorForTogether`:
|
| 248 |
+
- `pixel_values/image_grid_thw` 改成可选字段(存在才 stack)。
|
| 249 |
+
|
| 250 |
+
3. `Psi0Model.forward`:
|
| 251 |
+
- `pixel_values=None, image_grid_thw=None` 时也能调用 VLM。
|
| 252 |
+
- 对应 forward 参数改成 optional。
|
| 253 |
+
|
| 254 |
+
这是更“干净”的 text-to-motion 路径,但改动比 A 稍大。
|
| 255 |
+
|
| 256 |
+
---
|
| 257 |
+
|
| 258 |
+
## 5) Qwen-2.5-Omni-3B 替换要点
|
| 259 |
+
|
| 260 |
+
当前 posttrain 写死了 Qwen3 类:
|
| 261 |
+
|
| 262 |
+
1. `src/psi/trainers/posttrain.py` 里 import `Qwen3VLForConditionalGeneration`。
|
| 263 |
+
2. `src/psi/models/psi0.py` 类型标注也是 `Qwen3VLForConditionalGeneration`。
|
| 264 |
+
|
| 265 |
+
要换 Omni,最小做法:
|
| 266 |
+
|
| 267 |
+
1. 在 posttrain 里改成可切换 VLM 构造:
|
| 268 |
+
- `Qwen2_5OmniForConditionalGeneration` + `Qwen2_5OmniProcessor`(或 Auto 类统一封装)。
|
| 269 |
+
|
| 270 |
+
2. 保持 `output_hidden_states=True`,继续取最后一层 hidden states 作为 action expert 条件。
|
| 271 |
+
|
| 272 |
+
3. 校对 hidden size,并同步 `--model.view_feature_dim`。
|
| 273 |
+
|
| 274 |
+
你当前环境可用性(已验证):
|
| 275 |
+
|
| 276 |
+
1. `.venv-psi` 下 `transformers==4.57.0`。
|
| 277 |
+
2. 可 import:
|
| 278 |
+
- `Qwen2_5OmniForConditionalGeneration`
|
| 279 |
+
- `Qwen2_5OmniProcessor`
|
| 280 |
+
|
| 281 |
+
建议你自己再做一次快速确认:
|
| 282 |
+
|
| 283 |
+
```bash
|
| 284 |
+
.venv-psi/bin/python - <<'PY'
|
| 285 |
+
from transformers import Qwen2_5OmniForConditionalGeneration, Qwen2_5OmniProcessor
|
| 286 |
+
print("ok")
|
| 287 |
+
PY
|
| 288 |
+
```
|
| 289 |
+
|
| 290 |
+
再确认 hidden size(用于 `view_feature_dim`):
|
| 291 |
+
|
| 292 |
+
```bash
|
| 293 |
+
.venv-psi/bin/python - <<'PY'
|
| 294 |
+
from transformers import AutoConfig
|
| 295 |
+
cfg = AutoConfig.from_pretrained("Qwen/Qwen2.5-Omni-3B")
|
| 296 |
+
print("thinker hidden:", getattr(getattr(cfg, "thinker_config", None), "hidden_size", None))
|
| 297 |
+
print("hidden:", getattr(cfg, "hidden_size", None))
|
| 298 |
+
PY
|
| 299 |
+
```
|
| 300 |
+
|
| 301 |
+
---
|
| 302 |
+
|
| 303 |
+
## 6) 从数据处理开始的落地步骤(建议执行顺序)
|
| 304 |
+
|
| 305 |
+
1. 先定动作语义:
|
| 306 |
+
- 你的 36 维是绝对量还是 delta。
|
| 307 |
+
- 如果你数据已经是重定向后的绝对关节+位姿,优先 `use_delta_actions=False`,先简化问题。
|
| 308 |
+
|
| 309 |
+
2. 做 dataset 适配(最关键):
|
| 310 |
+
- 先写 `BoneSeedDataset.__getitem__`,把每个样本变成统一结构。
|
| 311 |
+
- 确保 `states/actions` 维度稳定。
|
| 312 |
+
- caption 映射到 `instruction`。
|
| 313 |
+
|
| 314 |
+
3. 生成统计文件(归一化必需):
|
| 315 |
+
- `ActionStateTransform` 需要 `stat_path`。
|
| 316 |
+
- 至少包含 `action` 键下的 `min/max` 或 `q01/q99`,维度 36。
|
| 317 |
+
|
| 318 |
+
4. 新建专用 config 与训练脚本:
|
| 319 |
+
- 先复用 posttrain trainer,不要改训练循环。
|
| 320 |
+
- 参数先直接对齐:`action_dim=36`, `odim=36`, `pad_*_dim=36`。
|
| 321 |
+
|
| 322 |
+
5. 再换 VLM 到 Omni:
|
| 323 |
+
- 先只做“文本+占位图”跑通。
|
| 324 |
+
- 然后再做严格 text-only 可选字段改造。
|
| 325 |
+
|
| 326 |
+
6. 最后做 smoke test:
|
| 327 |
+
- 1-2 个 batch overfit(`overfit_single_batch=True`)先看 loss 能不能快速下降。
|
| 328 |
+
|
| 329 |
+
---
|
| 330 |
+
|
| 331 |
+
## 7) 你现在就要注意的风险点
|
| 332 |
+
|
| 333 |
+
1. `HEPosttrainRepackTransform` 里有明显 typo:
|
| 334 |
+
- `obs_mask": datpa["obs_mask"]`,`datpa` 未定义。
|
| 335 |
+
- 位置:`src/psi/config/transform.py:766`
|
| 336 |
+
- 如果你复用这段逻辑,先修掉。
|
| 337 |
+
|
| 338 |
+
2. 评估指标分组写死(对 36 维不语义一致):
|
| 339 |
+
- `np.split(..., [7,14,21])`
|
| 340 |
+
- 位置:`src/psi/trainers/posttrain.py:512-517`
|
| 341 |
+
- 训练不一定受影响,但日志会误导。
|
| 342 |
+
|
| 343 |
+
3. 当前 collator/forward 对视觉字段是强依赖:
|
| 344 |
+
- 不改的话,text-only 会在 batch 组装处报错。
|
| 345 |
+
- 位置:`src/psi/trainers/qwen3vl_mixin.py:115-125`, `src/psi/models/psi0.py:1604-1622`
|
| 346 |
+
|
| 347 |
+
4. `view_feature_dim` 和 VLM hidden size 必须一致:
|
| 348 |
+
- 不一致会在线性层处 shape mismatch。
|
| 349 |
+
- 位置:`src/psi/models/psi0.py:499`
|
| 350 |
+
|
| 351 |
+
---
|
| 352 |
+
|
| 353 |
+
## 8) 最小改造文件清单(建议)
|
| 354 |
+
|
| 355 |
+
建议新增(而不是改老逻辑):
|
| 356 |
+
|
| 357 |
+
1. `src/psi/data/humanoid/boneseed_dataset.py`
|
| 358 |
+
2. `src/psi/config/data_boneseed.py`
|
| 359 |
+
3. `src/psi/config/train/posttrain_boneseed_psi0_config.py`
|
| 360 |
+
4. `scripts/train/psi0/posttrain-boneseed-psi0.sh`
|
| 361 |
+
|
| 362 |
+
如要做严格 text-only,再改:
|
| 363 |
+
|
| 364 |
+
1. `src/psi/config/transform.py`(`Psi0ModelTransform`)
|
| 365 |
+
2. `src/psi/trainers/qwen3vl_mixin.py`(`PaddedCollatorForTogether`)
|
| 366 |
+
3. `src/psi/models/psi0.py`(`Psi0Model.forward` optional vision args)
|
| 367 |
+
4. `src/psi/trainers/posttrain.py`(VLM 类替换成 Omni)
|
| 368 |
+
|
| 369 |
+
---
|
| 370 |
+
|
| 371 |
+
## 9) 一句话执行策略
|
| 372 |
+
|
| 373 |
+
先用“占位图 + 冻结 VLM + 只训 action expert”把 36 维 caption->motion 流程跑通,再把视觉输入改成 truly optional(text-only),最后替换到 Qwen2.5-Omni 并校准 `view_feature_dim`。
|
| 374 |
+
|
posttrain-full-duplex-audio-command-canonical-motion.sh
ADDED
|
@@ -0,0 +1,27 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#!/usr/bin/env bash
|
| 2 |
+
set -Eeuo pipefail
|
| 3 |
+
|
| 4 |
+
PROJECT_ROOT=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
|
| 5 |
+
CANONICAL_ROOT=${HFD_CONTINUOUS_DATA_ROOT:-/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/videogen/hfd_audio_command_continuous_office_v1}
|
| 6 |
+
|
| 7 |
+
# Isolated data ablation. The formal launcher's data default remains the
|
| 8 |
+
# complete corpus; only this wrapper enables one exact motion per instruction.
|
| 9 |
+
export HFD_CANONICAL_MOTION_MANIFEST=${HFD_CANONICAL_MOTION_MANIFEST:-$CANONICAL_ROOT/manifests/canonical_motion_per_instruction_v1.json}
|
| 10 |
+
export HFD_MAX_TRAINING_STEPS=${HFD_MAX_TRAINING_STEPS:-1000000}
|
| 11 |
+
export HFD_WANDB_GROUP=${HFD_WANDB_GROUP:-hfd-canonical-motion-ablation-v1}
|
| 12 |
+
# Keep this a one-variable data ablation. The formal launcher now enables
|
| 13 |
+
# Stop persistence supervision, but this wrapper pins the pre-change Stop
|
| 14 |
+
# objective/timeline so its result isolates canonical-motion selection only.
|
| 15 |
+
export HFD_STOP_ENTER_LOSS_WEIGHT=0.0
|
| 16 |
+
export HFD_STOP_STAY_LOSS_WEIGHT=0.0
|
| 17 |
+
export HFD_STOP_RELEASE_LOSS_WEIGHT=0.0
|
| 18 |
+
export HFD_CONTROL_POST_IDLE_MAX_STEPS=0
|
| 19 |
+
export HFD_CONTROL_POST_IDLE_SAMPLING=legacy_uniform
|
| 20 |
+
|
| 21 |
+
if [[ ! -f "$HFD_CANONICAL_MOTION_MANIFEST" ]]; then
|
| 22 |
+
echo "[error] canonical-motion manifest is missing: $HFD_CANONICAL_MOTION_MANIFEST" >&2
|
| 23 |
+
echo " build it with scripts/data/build_canonical_motion_manifest.py" >&2
|
| 24 |
+
exit 2
|
| 25 |
+
fi
|
| 26 |
+
|
| 27 |
+
exec bash "$PROJECT_ROOT/posttrain-full-duplex-audio-command.sh" "$@"
|
posttrain-full-duplex-audio-command-overfit.sh
ADDED
|
@@ -0,0 +1,43 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#!/usr/bin/env bash
|
| 2 |
+
set -Eeuo pipefail
|
| 3 |
+
|
| 4 |
+
PROJECT_ROOT=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
|
| 5 |
+
|
| 6 |
+
# Memorize a fixed 50-source subset instead of testing generalization. The
|
| 7 |
+
# Model, full-source cache scope, RTC timing and WandB authentication remain
|
| 8 |
+
# identical to formal HFDv2. Only the optimizer geometry is deliberately
|
| 9 |
+
# reduced so a fixed 50-source pool can produce real updates. Batch rows run
|
| 10 |
+
# together at each stream position rather than accumulating serially.
|
| 11 |
+
export HFD_OVERFIT_SOURCES=${HFD_OVERFIT_SOURCES:-50}
|
| 12 |
+
export HFD_LIMIT_SOURCES=$HFD_OVERFIT_SOURCES
|
| 13 |
+
export HFD_TRAIN_BATCH_SIZE=${HFD_TRAIN_BATCH_SIZE:-6}
|
| 14 |
+
export HFD_EFFECTIVE_SOURCES_PER_GPU=${HFD_EFFECTIVE_SOURCES_PER_GPU:-6}
|
| 15 |
+
export HFD_SOURCE_ACCUMULATION_GROUPS=${HFD_SOURCE_ACCUMULATION_GROUPS:-1}
|
| 16 |
+
# Keep the historical 50 independent sources intact. Version 13 partitions
|
| 17 |
+
# them into roughly 17 sessions, which cannot fill this eight-GPU B6 geometry.
|
| 18 |
+
export HFD_FULL_DUPLEX_SEMANTICS_VERSION=${HFD_FULL_DUPLEX_SEMANTICS_VERSION:-12}
|
| 19 |
+
export HFD_EXPERIMENT_MODE=overfit50
|
| 20 |
+
export HFD_VAL_BATCH_SIZE=${HFD_VAL_BATCH_SIZE:-2}
|
| 21 |
+
export HFD_MAX_TRAINING_STEPS=${HFD_MAX_TRAINING_STEPS:-50000}
|
| 22 |
+
export HFD_CHECKPOINTING_STEPS=${HFD_CHECKPOINTING_STEPS:-250}
|
| 23 |
+
export HFD_VALIDATION_STEPS=${HFD_VALIDATION_STEPS:-0}
|
| 24 |
+
export HFD_VAL_NUM_BATCHES=${HFD_VAL_NUM_BATCHES:-0}
|
| 25 |
+
export HFD_WANDB_GROUP=${HFD_WANDB_GROUP:-hfdv2-qwen-kv-overfit50}
|
| 26 |
+
export HFD_RUN_TIMESTAMP=${HFD_RUN_TIMESTAMP:-$(date +'%y%m%d%H%M')-hfdv2-overfit50}
|
| 27 |
+
|
| 28 |
+
# A 50-source overfit set supports at most six sources per GPU on eight GPUs
|
| 29 |
+
# (48 global). Singleton worker tasks let those six sources construct in
|
| 30 |
+
# parallel instead of serializing the whole batch in one worker.
|
| 31 |
+
export HFD_NUM_WORKERS=${HFD_NUM_WORKERS:-6}
|
| 32 |
+
export HFD_VAL_NUM_WORKERS=${HFD_VAL_NUM_WORKERS:-0}
|
| 33 |
+
export PSI_DATALOADER_PIN_MEMORY=${PSI_DATALOADER_PIN_MEMORY:-0}
|
| 34 |
+
|
| 35 |
+
echo "[overfit] fixed training sources: $HFD_LIMIT_SOURCES"
|
| 36 |
+
echo "[overfit] true parallel source batch per GPU: $HFD_TRAIN_BATCH_SIZE"
|
| 37 |
+
echo "[overfit] source accumulation groups: $HFD_SOURCE_ACCUMULATION_GROUPS"
|
| 38 |
+
echo "[overfit] effective sources per GPU: $HFD_EFFECTIVE_SOURCES_PER_GPU"
|
| 39 |
+
echo "[overfit] semantics version: $HFD_FULL_DUPLEX_SEMANTICS_VERSION"
|
| 40 |
+
echo "[overfit] validation disabled; judge memorization from train/loss"
|
| 41 |
+
echo "[overfit] WandB group: $HFD_WANDB_GROUP"
|
| 42 |
+
|
| 43 |
+
exec "$PROJECT_ROOT/posttrain-full-duplex-audio-command.sh" "$@"
|
posttrain-full-duplex-audio-command-smooth.sh
ADDED
|
@@ -0,0 +1,20 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#!/usr/bin/env bash
|
| 2 |
+
set -Eeuo pipefail
|
| 3 |
+
|
| 4 |
+
PROJECT_ROOT=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
|
| 5 |
+
|
| 6 |
+
# Version 11 keeps the version-9 stride-1 data/model/window-step optimizer and
|
| 7 |
+
# adds RTC-boundary-aware temporal derivative matching on the predicted clean
|
| 8 |
+
# TextOp39 action. The main launcher already defaults to this combined path;
|
| 9 |
+
# this wrapper remains as an explicit, readable entry point.
|
| 10 |
+
export HFD_FULL_DUPLEX_SEMANTICS_VERSION=11
|
| 11 |
+
export HFD_JOINT_VELOCITY_LOSS_WEIGHT=${HFD_JOINT_VELOCITY_LOSS_WEIGHT:-0.10}
|
| 12 |
+
export HFD_JOINT_ACCELERATION_LOSS_WEIGHT=${HFD_JOINT_ACCELERATION_LOSS_WEIGHT:-0.05}
|
| 13 |
+
export HFD_ROOT_CONTINUITY_LOSS_WEIGHT=${HFD_ROOT_CONTINUITY_LOSS_WEIGHT:-0.10}
|
| 14 |
+
export HFD_WANDB_GROUP=${HFD_WANDB_GROUP:-hfd-dense-stride1-qwen-stride20-rtc-heading-smooth-v11}
|
| 15 |
+
export HFD_RUN_TIMESTAMP=${HFD_RUN_TIMESTAMP:-$(date +'%y%m%d%H%M')-stride1-smooth-v11}
|
| 16 |
+
|
| 17 |
+
echo "[smooth-v11] stride-1 clean-action temporal derivative matching enabled"
|
| 18 |
+
echo "[smooth-v11] weights: joint_velocity=$HFD_JOINT_VELOCITY_LOSS_WEIGHT joint_acceleration=$HFD_JOINT_ACCELERATION_LOSS_WEIGHT root_continuity=$HFD_ROOT_CONTINUITY_LOSS_WEIGHT"
|
| 19 |
+
|
| 20 |
+
exec "$PROJECT_ROOT/posttrain-full-duplex-audio-command.sh" "$@"
|
posttrain-full-duplex-audio-command.sh
ADDED
|
@@ -0,0 +1,1523 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#!/usr/bin/env bash
|
| 2 |
+
set -Eeuo pipefail
|
| 3 |
+
|
| 4 |
+
PROJECT_ROOT=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
|
| 5 |
+
cd "$PROJECT_ROOT"
|
| 6 |
+
|
| 7 |
+
export PYTHONPATH="$PROJECT_ROOT/src${PYTHONPATH:+:$PYTHONPATH}"
|
| 8 |
+
export CUDA_VISIBLE_DEVICES=${CUDA_VISIBLE_DEVICES:-0,1,2,3,4,5,6,7}
|
| 9 |
+
export PSI_SKIP_VLM_MODEL=0
|
| 10 |
+
export HFD_MANUAL_DDP_SYNC=1
|
| 11 |
+
export TORCH_NCCL_ASYNC_ERROR_HANDLING=${TORCH_NCCL_ASYNC_ERROR_HANDLING:-1}
|
| 12 |
+
export PSI_FLOW_SIGMA_MIN=${PSI_FLOW_SIGMA_MIN:-0.05}
|
| 13 |
+
export PSI_FLOW_SIGMA_MAX=${PSI_FLOW_SIGMA_MAX:-1.0}
|
| 14 |
+
export PSI_DATALOADER_PIN_MEMORY=${PSI_DATALOADER_PIN_MEMORY:-1}
|
| 15 |
+
export PSI_WORKER_TORCH_THREADS=${PSI_WORKER_TORCH_THREADS:-1}
|
| 16 |
+
export PSI_FOREACH_GRAD_NORM=${PSI_FOREACH_GRAD_NORM:-1}
|
| 17 |
+
export PYTORCH_CUDA_ALLOC_CONF=${PYTORCH_CUDA_ALLOC_CONF:-expandable_segments:True}
|
| 18 |
+
export HFD_DDP_GRAD_BUCKET_MB=${HFD_DDP_GRAD_BUCKET_MB:-256}
|
| 19 |
+
|
| 20 |
+
source "$PROJECT_ROOT/scripts/activate_psi_env.sh"
|
| 21 |
+
HFD_ENV_FILE=${HFD_ENV_FILE:-$PROJECT_ROOT/.env}
|
| 22 |
+
if [[ -f "$HFD_ENV_FILE" ]]; then
|
| 23 |
+
set -a
|
| 24 |
+
source "$HFD_ENV_FILE"
|
| 25 |
+
set +a
|
| 26 |
+
fi
|
| 27 |
+
|
| 28 |
+
CONTINUOUS_DATA_ROOT=${HFD_CONTINUOUS_DATA_ROOT:-/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/videogen/hfd_audio_command_continuous_office_v1}
|
| 29 |
+
BACKGROUND_ASSIGNMENT_PATH=${HFD_BACKGROUND_ASSIGNMENT_PATH:-$CONTINUOUS_DATA_ROOT/manifests/source_background_assignments.jsonl}
|
| 30 |
+
BACKGROUND_MANIFEST=$CONTINUOUS_DATA_ROOT/manifests/background_sessions.jsonl
|
| 31 |
+
PRODUCTION_MARKER=${HFD_CONTINUOUS_PRODUCTION_MARKER:-$CONTINUOUS_DATA_ROOT/reports/production_complete.json}
|
| 32 |
+
MOTION_ROOT=${HFD_MOTION_ROOT:-/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/Psi0/dataset/seed_all}
|
| 33 |
+
MOTION_METADATA=${HFD_MOTION_METADATA:-$MOTION_ROOT/metadata/seed_metadata_v004.csv}
|
| 34 |
+
AUDIO_MANIFEST=${HFD_AUDIO_COMMAND_MANIFEST:-/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/qwen3-tts/metadata.jsonl}
|
| 35 |
+
CANONICAL_MOTION_MANIFEST=${HFD_CANONICAL_MOTION_MANIFEST:-}
|
| 36 |
+
BACKBONE=${HFD_QWEN_BACKBONE:-/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/team_workspace/public_ckpt/Qwen/Qwen2.5-Omni-3B}
|
| 37 |
+
OUTPUT_DIR=${HFD_OUTPUT_DIR:-$PROJECT_ROOT/runs}
|
| 38 |
+
REPORT_TO=${HFD_REPORT_TO:-wandb}
|
| 39 |
+
if [[ "${REPORT_TO,,}" == "none" ]]; then
|
| 40 |
+
REPORT_TO=None
|
| 41 |
+
fi
|
| 42 |
+
WANDB_PROJECT=${HFD_WANDB_PROJECT:-humanoid-full-duplex}
|
| 43 |
+
WANDB_GROUP=${HFD_WANDB_GROUP:-}
|
| 44 |
+
EXPERIMENT_MODE=${HFD_EXPERIMENT_MODE:-formal}
|
| 45 |
+
|
| 46 |
+
FULL_DUPLEX_SEMANTICS_VERSION=${HFD_FULL_DUPLEX_SEMANTICS_VERSION:-24}
|
| 47 |
+
if [[ "$FULL_DUPLEX_SEMANTICS_VERSION" == "23" \
|
| 48 |
+
|| "$FULL_DUPLEX_SEMANTICS_VERSION" == "24" ]]; then
|
| 49 |
+
MAX_TRAINING_STEPS=${HFD_MAX_TRAINING_STEPS:-1250000}
|
| 50 |
+
else
|
| 51 |
+
MAX_TRAINING_STEPS=${HFD_MAX_TRAINING_STEPS:-1000000}
|
| 52 |
+
fi
|
| 53 |
+
CHECKPOINTING_STEPS=${HFD_CHECKPOINTING_STEPS:-5000}
|
| 54 |
+
# Validation runs only after a complete source group and samples only the
|
| 55 |
+
# 0.4-second Qwen/deployment cadence. Sixty-four sources are therefore about
|
| 56 |
+
# five times cheaper than the old 16-source dense stride-1 validation while
|
| 57 |
+
# giving checkpoint selection a broader, deterministic held-out sample.
|
| 58 |
+
# Validation is synchronous and pauses all training GPUs. Checkpoints remain
|
| 59 |
+
# every 5k for recovery/auto-infer, while the fixed held-out suite runs every
|
| 60 |
+
# 20k by default; callers can still request a denser diagnostic cadence.
|
| 61 |
+
VALIDATION_STEPS=${HFD_VALIDATION_STEPS:-20000}
|
| 62 |
+
VAL_NUM_BATCHES=${HFD_VAL_NUM_BATCHES:-64}
|
| 63 |
+
BEST_CHECKPOINT_MIN_RELATIVE_DELTA=${HFD_BEST_CHECKPOINT_MIN_RELATIVE_DELTA:-0.002}
|
| 64 |
+
TRAIN_BATCH_SIZE=${HFD_TRAIN_BATCH_SIZE:-192}
|
| 65 |
+
EFFECTIVE_SOURCES_PER_GPU=${HFD_EFFECTIVE_SOURCES_PER_GPU:-192}
|
| 66 |
+
SOURCE_ACCUMULATION_GROUPS=${HFD_SOURCE_ACCUMULATION_GROUPS:-}
|
| 67 |
+
QWEN_KV_GRADIENT_HORIZON=${HFD_QWEN_KV_GRADIENT_HORIZON:-1}
|
| 68 |
+
QWEN_TRAINABLE_LAST_LAYERS=${HFD_QWEN_TRAINABLE_LAST_LAYERS:-2}
|
| 69 |
+
QWEN_LR=${HFD_QWEN_LR:-2e-6}
|
| 70 |
+
MIN_LR_RATE=${HFD_MIN_LR_RATE:-0.05}
|
| 71 |
+
ACTION_RECOMPUTE=${HFD_ACTION_RECOMPUTE:-0}
|
| 72 |
+
VAL_BATCH_SIZE=${HFD_VAL_BATCH_SIZE:-8}
|
| 73 |
+
SOURCE_LENGTH_LOOKAHEAD_BATCHES=${HFD_SOURCE_LENGTH_LOOKAHEAD_BATCHES:-4}
|
| 74 |
+
COMPACT_INACTIVE_ACTION_ROWS=${HFD_COMPACT_INACTIVE_ACTION_ROWS:-1}
|
| 75 |
+
DYNAMIC_SOURCE_REFILL_EXPLICIT=${HFD_DYNAMIC_SOURCE_REFILL+x}
|
| 76 |
+
DYNAMIC_SOURCE_REFILL=${HFD_DYNAMIC_SOURCE_REFILL:-1}
|
| 77 |
+
DYNAMIC_SOURCE_CONTEXT_RESERVE_TOKENS=${HFD_DYNAMIC_SOURCE_CONTEXT_RESERVE_TOKENS:-128}
|
| 78 |
+
GRADIENT_ACCUMULATION_STEPS=${HFD_GRADIENT_ACCUMULATION_STEPS:-1}
|
| 79 |
+
NUM_WORKERS=${HFD_NUM_WORKERS:-16}
|
| 80 |
+
VAL_NUM_WORKERS=${HFD_VAL_NUM_WORKERS:-0}
|
| 81 |
+
TBPTT_UNROLL_STEPS=${HFD_TBPTT_UNROLL_STEPS:-0}
|
| 82 |
+
CONTROL_LOSS_WEIGHT=${HFD_CONTROL_LOSS_WEIGHT:-1.0}
|
| 83 |
+
CONTROL_HEAD_LR_MULTIPLIER_EXPLICIT=${HFD_CONTROL_HEAD_LR_MULTIPLIER+x}
|
| 84 |
+
CONTROL_HEAD_LR_MULTIPLIER=${HFD_CONTROL_HEAD_LR_MULTIPLIER:-0.25}
|
| 85 |
+
MOTION_DURATION_DEFAULT_SECONDS=${HFD_MOTION_DURATION_DEFAULT_SECONDS:-8.0}
|
| 86 |
+
MOTION_DURATION_DISTRIBUTION_BINS=${HFD_MOTION_DURATION_DISTRIBUTION_BINS:-1}
|
| 87 |
+
MOTION_DURATION_MIN_SECONDS=${HFD_MOTION_DURATION_MIN_SECONDS:-0.5}
|
| 88 |
+
MOTION_DURATION_MAX_SECONDS=${HFD_MOTION_DURATION_MAX_SECONDS:-180.0}
|
| 89 |
+
DENSE_WINDOW_ACCUMULATION_STEPS=${HFD_DENSE_WINDOW_ACCUMULATION_STEPS:-1}
|
| 90 |
+
STATE_AWARE_CONTROL_MODEL_ARG=--model.no-state-aware-serial-control
|
| 91 |
+
if [[ "$FULL_DUPLEX_SEMANTICS_VERSION" == "24" ]]; then
|
| 92 |
+
JOINT_VELOCITY_LOSS_WEIGHT=${HFD_JOINT_VELOCITY_LOSS_WEIGHT:-0.10}
|
| 93 |
+
JOINT_ACCELERATION_LOSS_WEIGHT=${HFD_JOINT_ACCELERATION_LOSS_WEIGHT:-0.05}
|
| 94 |
+
ROOT_CONTINUITY_LOSS_WEIGHT=${HFD_ROOT_CONTINUITY_LOSS_WEIGHT:-0.10}
|
| 95 |
+
STATIONARY_ROOT_HEIGHT_LOSS_WEIGHT=${HFD_STATIONARY_ROOT_HEIGHT_LOSS_WEIGHT:-1.0}
|
| 96 |
+
MOTION_DURATION_LOSS_WEIGHT=${HFD_MOTION_DURATION_LOSS_WEIGHT:-0.20}
|
| 97 |
+
LATEST_INSTRUCTION_LOSS_WEIGHT=${HFD_LATEST_INSTRUCTION_LOSS_WEIGHT:-0.10}
|
| 98 |
+
MOTION_DURATION_MODEL_ARG=--model.predict-motion-duration
|
| 99 |
+
MOTION_DURATION_DISTRIBUTION_BINS=${HFD_MOTION_DURATION_DISTRIBUTION_BINS:-48}
|
| 100 |
+
DENSE_WINDOW_ACCUMULATION_STEPS=${HFD_DENSE_WINDOW_ACCUMULATION_STEPS:-1}
|
| 101 |
+
WANDB_GROUP=${WANDB_GROUP:-hfd-recency-memory-v24}
|
| 102 |
+
elif [[ "$FULL_DUPLEX_SEMANTICS_VERSION" == "23" ]]; then
|
| 103 |
+
JOINT_VELOCITY_LOSS_WEIGHT=${HFD_JOINT_VELOCITY_LOSS_WEIGHT:-0.10}
|
| 104 |
+
JOINT_ACCELERATION_LOSS_WEIGHT=${HFD_JOINT_ACCELERATION_LOSS_WEIGHT:-0.05}
|
| 105 |
+
ROOT_CONTINUITY_LOSS_WEIGHT=${HFD_ROOT_CONTINUITY_LOSS_WEIGHT:-0.10}
|
| 106 |
+
STATIONARY_ROOT_HEIGHT_LOSS_WEIGHT=${HFD_STATIONARY_ROOT_HEIGHT_LOSS_WEIGHT:-1.0}
|
| 107 |
+
MOTION_DURATION_LOSS_WEIGHT=${HFD_MOTION_DURATION_LOSS_WEIGHT:-0.20}
|
| 108 |
+
LATEST_INSTRUCTION_LOSS_WEIGHT=${HFD_LATEST_INSTRUCTION_LOSS_WEIGHT:-0.10}
|
| 109 |
+
MOTION_DURATION_MODEL_ARG=--model.predict-motion-duration
|
| 110 |
+
MOTION_DURATION_DISTRIBUTION_BINS=${HFD_MOTION_DURATION_DISTRIBUTION_BINS:-48}
|
| 111 |
+
DENSE_WINDOW_ACCUMULATION_STEPS=${HFD_DENSE_WINDOW_ACCUMULATION_STEPS:-1}
|
| 112 |
+
WANDB_GROUP=${WANDB_GROUP:-hfd-hesitation-pause-v23}
|
| 113 |
+
elif [[ "$FULL_DUPLEX_SEMANTICS_VERSION" == "22" ]]; then
|
| 114 |
+
JOINT_VELOCITY_LOSS_WEIGHT=${HFD_JOINT_VELOCITY_LOSS_WEIGHT:-0.10}
|
| 115 |
+
JOINT_ACCELERATION_LOSS_WEIGHT=${HFD_JOINT_ACCELERATION_LOSS_WEIGHT:-0.05}
|
| 116 |
+
ROOT_CONTINUITY_LOSS_WEIGHT=${HFD_ROOT_CONTINUITY_LOSS_WEIGHT:-0.10}
|
| 117 |
+
STATIONARY_ROOT_HEIGHT_LOSS_WEIGHT=${HFD_STATIONARY_ROOT_HEIGHT_LOSS_WEIGHT:-1.0}
|
| 118 |
+
MOTION_DURATION_LOSS_WEIGHT=${HFD_MOTION_DURATION_LOSS_WEIGHT:-0.20}
|
| 119 |
+
LATEST_INSTRUCTION_LOSS_WEIGHT=${HFD_LATEST_INSTRUCTION_LOSS_WEIGHT:-0.10}
|
| 120 |
+
MOTION_DURATION_MODEL_ARG=--model.predict-motion-duration
|
| 121 |
+
MOTION_DURATION_DISTRIBUTION_BINS=${HFD_MOTION_DURATION_DISTRIBUTION_BINS:-48}
|
| 122 |
+
DENSE_WINDOW_ACCUMULATION_STEPS=${HFD_DENSE_WINDOW_ACCUMULATION_STEPS:-1}
|
| 123 |
+
WANDB_GROUP=${WANDB_GROUP:-hfd-semantic-control-duration-distribution-v22}
|
| 124 |
+
elif [[ "$FULL_DUPLEX_SEMANTICS_VERSION" == "21" ]]; then
|
| 125 |
+
JOINT_VELOCITY_LOSS_WEIGHT=${HFD_JOINT_VELOCITY_LOSS_WEIGHT:-0.10}
|
| 126 |
+
JOINT_ACCELERATION_LOSS_WEIGHT=${HFD_JOINT_ACCELERATION_LOSS_WEIGHT:-0.05}
|
| 127 |
+
ROOT_CONTINUITY_LOSS_WEIGHT=${HFD_ROOT_CONTINUITY_LOSS_WEIGHT:-0.10}
|
| 128 |
+
STATIONARY_ROOT_HEIGHT_LOSS_WEIGHT=${HFD_STATIONARY_ROOT_HEIGHT_LOSS_WEIGHT:-1.0}
|
| 129 |
+
MOTION_DURATION_LOSS_WEIGHT=${HFD_MOTION_DURATION_LOSS_WEIGHT:-0.20}
|
| 130 |
+
LATEST_INSTRUCTION_LOSS_WEIGHT=${HFD_LATEST_INSTRUCTION_LOSS_WEIGHT:-0.10}
|
| 131 |
+
MOTION_DURATION_MODEL_ARG=--model.predict-motion-duration
|
| 132 |
+
STATE_AWARE_CONTROL_MODEL_ARG=--model.state-aware-serial-control
|
| 133 |
+
WANDB_GROUP=${WANDB_GROUP:-hfd-state-control-full-duplex-v21}
|
| 134 |
+
elif [[ "$FULL_DUPLEX_SEMANTICS_VERSION" == "20" ]]; then
|
| 135 |
+
JOINT_VELOCITY_LOSS_WEIGHT=${HFD_JOINT_VELOCITY_LOSS_WEIGHT:-0.10}
|
| 136 |
+
JOINT_ACCELERATION_LOSS_WEIGHT=${HFD_JOINT_ACCELERATION_LOSS_WEIGHT:-0.05}
|
| 137 |
+
ROOT_CONTINUITY_LOSS_WEIGHT=${HFD_ROOT_CONTINUITY_LOSS_WEIGHT:-0.10}
|
| 138 |
+
STATIONARY_ROOT_HEIGHT_LOSS_WEIGHT=${HFD_STATIONARY_ROOT_HEIGHT_LOSS_WEIGHT:-1.0}
|
| 139 |
+
MOTION_DURATION_LOSS_WEIGHT=${HFD_MOTION_DURATION_LOSS_WEIGHT:-0.20}
|
| 140 |
+
LATEST_INSTRUCTION_LOSS_WEIGHT=${HFD_LATEST_INSTRUCTION_LOSS_WEIGHT:-0.10}
|
| 141 |
+
MOTION_DURATION_MODEL_ARG=--model.predict-motion-duration
|
| 142 |
+
STATE_AWARE_CONTROL_MODEL_ARG=--model.no-state-aware-serial-control
|
| 143 |
+
WANDB_GROUP=${WANDB_GROUP:-hfd-interruption-v20}
|
| 144 |
+
elif [[ "$FULL_DUPLEX_SEMANTICS_VERSION" == "19" ]]; then
|
| 145 |
+
JOINT_VELOCITY_LOSS_WEIGHT=${HFD_JOINT_VELOCITY_LOSS_WEIGHT:-0.10}
|
| 146 |
+
JOINT_ACCELERATION_LOSS_WEIGHT=${HFD_JOINT_ACCELERATION_LOSS_WEIGHT:-0.05}
|
| 147 |
+
ROOT_CONTINUITY_LOSS_WEIGHT=${HFD_ROOT_CONTINUITY_LOSS_WEIGHT:-0.10}
|
| 148 |
+
STATIONARY_ROOT_HEIGHT_LOSS_WEIGHT=${HFD_STATIONARY_ROOT_HEIGHT_LOSS_WEIGHT:-1.0}
|
| 149 |
+
MOTION_DURATION_LOSS_WEIGHT=${HFD_MOTION_DURATION_LOSS_WEIGHT:-0.20}
|
| 150 |
+
LATEST_INSTRUCTION_LOSS_WEIGHT=${HFD_LATEST_INSTRUCTION_LOSS_WEIGHT:-0.10}
|
| 151 |
+
MOTION_DURATION_MODEL_ARG=--model.predict-motion-duration
|
| 152 |
+
WANDB_GROUP=${WANDB_GROUP:-hfd-rtc-transition-v19}
|
| 153 |
+
elif [[ "$FULL_DUPLEX_SEMANTICS_VERSION" == "18" ]]; then
|
| 154 |
+
JOINT_VELOCITY_LOSS_WEIGHT=${HFD_JOINT_VELOCITY_LOSS_WEIGHT:-0.10}
|
| 155 |
+
JOINT_ACCELERATION_LOSS_WEIGHT=${HFD_JOINT_ACCELERATION_LOSS_WEIGHT:-0.05}
|
| 156 |
+
ROOT_CONTINUITY_LOSS_WEIGHT=${HFD_ROOT_CONTINUITY_LOSS_WEIGHT:-0.10}
|
| 157 |
+
STATIONARY_ROOT_HEIGHT_LOSS_WEIGHT=${HFD_STATIONARY_ROOT_HEIGHT_LOSS_WEIGHT:-1.0}
|
| 158 |
+
MOTION_DURATION_LOSS_WEIGHT=${HFD_MOTION_DURATION_LOSS_WEIGHT:-0.20}
|
| 159 |
+
LATEST_INSTRUCTION_LOSS_WEIGHT=${HFD_LATEST_INSTRUCTION_LOSS_WEIGHT:-0.10}
|
| 160 |
+
MOTION_DURATION_MODEL_ARG=--model.predict-motion-duration
|
| 161 |
+
WANDB_GROUP=${WANDB_GROUP:-hfd-latest-command-v18}
|
| 162 |
+
elif [[ "$FULL_DUPLEX_SEMANTICS_VERSION" == "17" ]]; then
|
| 163 |
+
JOINT_VELOCITY_LOSS_WEIGHT=${HFD_JOINT_VELOCITY_LOSS_WEIGHT:-0.10}
|
| 164 |
+
JOINT_ACCELERATION_LOSS_WEIGHT=${HFD_JOINT_ACCELERATION_LOSS_WEIGHT:-0.05}
|
| 165 |
+
ROOT_CONTINUITY_LOSS_WEIGHT=${HFD_ROOT_CONTINUITY_LOSS_WEIGHT:-0.10}
|
| 166 |
+
STATIONARY_ROOT_HEIGHT_LOSS_WEIGHT=${HFD_STATIONARY_ROOT_HEIGHT_LOSS_WEIGHT:-1.0}
|
| 167 |
+
MOTION_DURATION_LOSS_WEIGHT=${HFD_MOTION_DURATION_LOSS_WEIGHT:-0.20}
|
| 168 |
+
LATEST_INSTRUCTION_LOSS_WEIGHT=${HFD_LATEST_INSTRUCTION_LOSS_WEIGHT:-0.0}
|
| 169 |
+
MOTION_DURATION_MODEL_ARG=--model.predict-motion-duration
|
| 170 |
+
WANDB_GROUP=${WANDB_GROUP:-hfd-session-voice-v17}
|
| 171 |
+
elif [[ "$FULL_DUPLEX_SEMANTICS_VERSION" == "16" ]]; then
|
| 172 |
+
JOINT_VELOCITY_LOSS_WEIGHT=${HFD_JOINT_VELOCITY_LOSS_WEIGHT:-0.10}
|
| 173 |
+
JOINT_ACCELERATION_LOSS_WEIGHT=${HFD_JOINT_ACCELERATION_LOSS_WEIGHT:-0.05}
|
| 174 |
+
ROOT_CONTINUITY_LOSS_WEIGHT=${HFD_ROOT_CONTINUITY_LOSS_WEIGHT:-0.10}
|
| 175 |
+
STATIONARY_ROOT_HEIGHT_LOSS_WEIGHT=${HFD_STATIONARY_ROOT_HEIGHT_LOSS_WEIGHT:-1.0}
|
| 176 |
+
MOTION_DURATION_LOSS_WEIGHT=${HFD_MOTION_DURATION_LOSS_WEIGHT:-0.20}
|
| 177 |
+
LATEST_INSTRUCTION_LOSS_WEIGHT=${HFD_LATEST_INSTRUCTION_LOSS_WEIGHT:-0.0}
|
| 178 |
+
MOTION_DURATION_MODEL_ARG=--model.predict-motion-duration
|
| 179 |
+
WANDB_GROUP=${WANDB_GROUP:-hfd-motion-duration-v16}
|
| 180 |
+
elif [[ "$FULL_DUPLEX_SEMANTICS_VERSION" == "15" ]]; then
|
| 181 |
+
JOINT_VELOCITY_LOSS_WEIGHT=${HFD_JOINT_VELOCITY_LOSS_WEIGHT:-0.10}
|
| 182 |
+
JOINT_ACCELERATION_LOSS_WEIGHT=${HFD_JOINT_ACCELERATION_LOSS_WEIGHT:-0.05}
|
| 183 |
+
ROOT_CONTINUITY_LOSS_WEIGHT=${HFD_ROOT_CONTINUITY_LOSS_WEIGHT:-0.10}
|
| 184 |
+
STATIONARY_ROOT_HEIGHT_LOSS_WEIGHT=${HFD_STATIONARY_ROOT_HEIGHT_LOSS_WEIGHT:-1.0}
|
| 185 |
+
MOTION_DURATION_LOSS_WEIGHT=${HFD_MOTION_DURATION_LOSS_WEIGHT:-0.0}
|
| 186 |
+
LATEST_INSTRUCTION_LOSS_WEIGHT=${HFD_LATEST_INSTRUCTION_LOSS_WEIGHT:-0.0}
|
| 187 |
+
MOTION_DURATION_MODEL_ARG=--model.no-predict-motion-duration
|
| 188 |
+
WANDB_GROUP=${WANDB_GROUP:-hfd-cross-window-smooth-heading-v15}
|
| 189 |
+
elif [[ "$FULL_DUPLEX_SEMANTICS_VERSION" == "14" ]]; then
|
| 190 |
+
JOINT_VELOCITY_LOSS_WEIGHT=${HFD_JOINT_VELOCITY_LOSS_WEIGHT:-0.10}
|
| 191 |
+
JOINT_ACCELERATION_LOSS_WEIGHT=${HFD_JOINT_ACCELERATION_LOSS_WEIGHT:-0.05}
|
| 192 |
+
ROOT_CONTINUITY_LOSS_WEIGHT=${HFD_ROOT_CONTINUITY_LOSS_WEIGHT:-0.10}
|
| 193 |
+
STATIONARY_ROOT_HEIGHT_LOSS_WEIGHT=${HFD_STATIONARY_ROOT_HEIGHT_LOSS_WEIGHT:-1.0}
|
| 194 |
+
MOTION_DURATION_LOSS_WEIGHT=${HFD_MOTION_DURATION_LOSS_WEIGHT:-0.0}
|
| 195 |
+
LATEST_INSTRUCTION_LOSS_WEIGHT=${HFD_LATEST_INSTRUCTION_LOSS_WEIGHT:-0.0}
|
| 196 |
+
MOTION_DURATION_MODEL_ARG=--model.no-predict-motion-duration
|
| 197 |
+
WANDB_GROUP=${WANDB_GROUP:-hfd-multiturn-causal-listen-v14}
|
| 198 |
+
elif [[ "$FULL_DUPLEX_SEMANTICS_VERSION" == "13" ]]; then
|
| 199 |
+
JOINT_VELOCITY_LOSS_WEIGHT=${HFD_JOINT_VELOCITY_LOSS_WEIGHT:-0.10}
|
| 200 |
+
JOINT_ACCELERATION_LOSS_WEIGHT=${HFD_JOINT_ACCELERATION_LOSS_WEIGHT:-0.05}
|
| 201 |
+
ROOT_CONTINUITY_LOSS_WEIGHT=${HFD_ROOT_CONTINUITY_LOSS_WEIGHT:-0.10}
|
| 202 |
+
STATIONARY_ROOT_HEIGHT_LOSS_WEIGHT=${HFD_STATIONARY_ROOT_HEIGHT_LOSS_WEIGHT:-1.0}
|
| 203 |
+
MOTION_DURATION_LOSS_WEIGHT=${HFD_MOTION_DURATION_LOSS_WEIGHT:-0.0}
|
| 204 |
+
LATEST_INSTRUCTION_LOSS_WEIGHT=${HFD_LATEST_INSTRUCTION_LOSS_WEIGHT:-0.0}
|
| 205 |
+
MOTION_DURATION_MODEL_ARG=--model.no-predict-motion-duration
|
| 206 |
+
WANDB_GROUP=${WANDB_GROUP:-hfd-multiturn-session-v13}
|
| 207 |
+
elif [[ "$FULL_DUPLEX_SEMANTICS_VERSION" == "12" ]]; then
|
| 208 |
+
JOINT_VELOCITY_LOSS_WEIGHT=${HFD_JOINT_VELOCITY_LOSS_WEIGHT:-0.10}
|
| 209 |
+
JOINT_ACCELERATION_LOSS_WEIGHT=${HFD_JOINT_ACCELERATION_LOSS_WEIGHT:-0.05}
|
| 210 |
+
ROOT_CONTINUITY_LOSS_WEIGHT=${HFD_ROOT_CONTINUITY_LOSS_WEIGHT:-0.10}
|
| 211 |
+
STATIONARY_ROOT_HEIGHT_LOSS_WEIGHT=${HFD_STATIONARY_ROOT_HEIGHT_LOSS_WEIGHT:-1.0}
|
| 212 |
+
MOTION_DURATION_LOSS_WEIGHT=${HFD_MOTION_DURATION_LOSS_WEIGHT:-0.0}
|
| 213 |
+
LATEST_INSTRUCTION_LOSS_WEIGHT=${HFD_LATEST_INSTRUCTION_LOSS_WEIGHT:-0.0}
|
| 214 |
+
MOTION_DURATION_MODEL_ARG=--model.no-predict-motion-duration
|
| 215 |
+
WANDB_GROUP=${WANDB_GROUP:-hfd-framewise-control-ground-v12}
|
| 216 |
+
elif [[ "$FULL_DUPLEX_SEMANTICS_VERSION" == "11" ]]; then
|
| 217 |
+
JOINT_VELOCITY_LOSS_WEIGHT=${HFD_JOINT_VELOCITY_LOSS_WEIGHT:-0.10}
|
| 218 |
+
JOINT_ACCELERATION_LOSS_WEIGHT=${HFD_JOINT_ACCELERATION_LOSS_WEIGHT:-0.05}
|
| 219 |
+
ROOT_CONTINUITY_LOSS_WEIGHT=${HFD_ROOT_CONTINUITY_LOSS_WEIGHT:-0.10}
|
| 220 |
+
STATIONARY_ROOT_HEIGHT_LOSS_WEIGHT=${HFD_STATIONARY_ROOT_HEIGHT_LOSS_WEIGHT:-0.0}
|
| 221 |
+
MOTION_DURATION_LOSS_WEIGHT=${HFD_MOTION_DURATION_LOSS_WEIGHT:-0.0}
|
| 222 |
+
LATEST_INSTRUCTION_LOSS_WEIGHT=${HFD_LATEST_INSTRUCTION_LOSS_WEIGHT:-0.0}
|
| 223 |
+
MOTION_DURATION_MODEL_ARG=--model.no-predict-motion-duration
|
| 224 |
+
WANDB_GROUP=${WANDB_GROUP:-hfd-dense-stride1-qwen-stride20-rtc-heading-smooth-v11}
|
| 225 |
+
elif [[ "$FULL_DUPLEX_SEMANTICS_VERSION" == "9" ]]; then
|
| 226 |
+
JOINT_VELOCITY_LOSS_WEIGHT=${HFD_JOINT_VELOCITY_LOSS_WEIGHT:-0.0}
|
| 227 |
+
JOINT_ACCELERATION_LOSS_WEIGHT=${HFD_JOINT_ACCELERATION_LOSS_WEIGHT:-0.0}
|
| 228 |
+
ROOT_CONTINUITY_LOSS_WEIGHT=${HFD_ROOT_CONTINUITY_LOSS_WEIGHT:-0.0}
|
| 229 |
+
STATIONARY_ROOT_HEIGHT_LOSS_WEIGHT=${HFD_STATIONARY_ROOT_HEIGHT_LOSS_WEIGHT:-0.0}
|
| 230 |
+
MOTION_DURATION_LOSS_WEIGHT=${HFD_MOTION_DURATION_LOSS_WEIGHT:-0.0}
|
| 231 |
+
LATEST_INSTRUCTION_LOSS_WEIGHT=${HFD_LATEST_INSTRUCTION_LOSS_WEIGHT:-0.0}
|
| 232 |
+
MOTION_DURATION_MODEL_ARG=--model.no-predict-motion-duration
|
| 233 |
+
WANDB_GROUP=${WANDB_GROUP:-hfd-dense-stride1-qwen-stride20-rtc}
|
| 234 |
+
else
|
| 235 |
+
echo "[error] HFD_FULL_DUPLEX_SEMANTICS_VERSION must be 9, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23 or 24." >&2
|
| 236 |
+
exit 2
|
| 237 |
+
fi
|
| 238 |
+
if [[ "$FULL_DUPLEX_SEMANTICS_VERSION" == "24" ]]; then
|
| 239 |
+
INSTRUCTION_RECENCY_SLOTS=${HFD_INSTRUCTION_RECENCY_SLOTS:-3}
|
| 240 |
+
INSTRUCTION_MEMORY_LOSS_WEIGHT=${HFD_INSTRUCTION_MEMORY_LOSS_WEIGHT:-0.05}
|
| 241 |
+
else
|
| 242 |
+
INSTRUCTION_RECENCY_SLOTS=${HFD_INSTRUCTION_RECENCY_SLOTS:-0}
|
| 243 |
+
INSTRUCTION_MEMORY_LOSS_WEIGHT=${HFD_INSTRUCTION_MEMORY_LOSS_WEIGHT:-0.0}
|
| 244 |
+
fi
|
| 245 |
+
CONTROL_CONDITION_TRAIN_MODE=${HFD_CONTROL_CONDITION_TRAIN_MODE:-hard}
|
| 246 |
+
CONTROL_CONDITION_INFERENCE_MODE=${HFD_CONTROL_CONDITION_INFERENCE_MODE:-hard}
|
| 247 |
+
CONTROL_CONDITION_TEMPERATURE=${HFD_CONTROL_CONDITION_TEMPERATURE:-1.0}
|
| 248 |
+
MAX_GRAD_NORM=${HFD_MAX_GRAD_NORM:-1.0}
|
| 249 |
+
LIMIT_SOURCES=${HFD_LIMIT_SOURCES:-0}
|
| 250 |
+
MAX_MOTION_STEPS=${HFD_MAX_MOTION_STEPS:-0}
|
| 251 |
+
STATIONARY_BLEND_FRAMES=${HFD_STATIONARY_BLEND_FRAMES:-20}
|
| 252 |
+
SESSION_MIN_TURNS=${HFD_SESSION_MIN_TURNS:-2}
|
| 253 |
+
SESSION_MAX_TURNS=${HFD_SESSION_MAX_TURNS:-4}
|
| 254 |
+
if [[ "$FULL_DUPLEX_SEMANTICS_VERSION" == "17" \
|
| 255 |
+
|| "$FULL_DUPLEX_SEMANTICS_VERSION" == "18" \
|
| 256 |
+
|| "$FULL_DUPLEX_SEMANTICS_VERSION" == "19" \
|
| 257 |
+
|| "$FULL_DUPLEX_SEMANTICS_VERSION" == "20" \
|
| 258 |
+
|| "$FULL_DUPLEX_SEMANTICS_VERSION" == "21" \
|
| 259 |
+
|| "$FULL_DUPLEX_SEMANTICS_VERSION" == "22" \
|
| 260 |
+
|| "$FULL_DUPLEX_SEMANTICS_VERSION" == "23" \
|
| 261 |
+
|| "$FULL_DUPLEX_SEMANTICS_VERSION" == "24" ]]; then
|
| 262 |
+
SESSION_VOICE_VARIANTS=${HFD_SESSION_VOICE_VARIANTS:-2}
|
| 263 |
+
else
|
| 264 |
+
SESSION_VOICE_VARIANTS=${HFD_SESSION_VOICE_VARIANTS:-0}
|
| 265 |
+
fi
|
| 266 |
+
TURN_TRANSITION_FRAMES=${HFD_TURN_TRANSITION_FRAMES:-50}
|
| 267 |
+
MAX_SPEECH_ONSET_STEPS=${HFD_MAX_SPEECH_ONSET_STEPS:-3}
|
| 268 |
+
LISTEN_REACTION_STEPS=${HFD_LISTEN_REACTION_STEPS:-1}
|
| 269 |
+
INTERRUPTION_RATIO=${HFD_INTERRUPTION_RATIO:-0.5}
|
| 270 |
+
INTERRUPTION_MIN_START_SECONDS=${HFD_INTERRUPTION_MIN_START_SECONDS:-2.0}
|
| 271 |
+
INTERRUPTION_MAX_START_SECONDS=${HFD_INTERRUPTION_MAX_START_SECONDS:-6.0}
|
| 272 |
+
INTERRUPTION_MIN_REMAINING_SECONDS=${HFD_INTERRUPTION_MIN_REMAINING_SECONDS:-1.5}
|
| 273 |
+
if [[ "$FULL_DUPLEX_SEMANTICS_VERSION" == "22" \
|
| 274 |
+
|| "$FULL_DUPLEX_SEMANTICS_VERSION" == "23" \
|
| 275 |
+
|| "$FULL_DUPLEX_SEMANTICS_VERSION" == "24" ]]; then
|
| 276 |
+
CONTROL_MIN_START_SECONDS=${HFD_CONTROL_MIN_START_SECONDS:-1.2}
|
| 277 |
+
CONTROL_MAX_START_SECONDS=${HFD_CONTROL_MAX_START_SECONDS:-4.0}
|
| 278 |
+
CONTROL_MIN_REMAINING_SECONDS=${HFD_CONTROL_MIN_REMAINING_SECONDS:-2.0}
|
| 279 |
+
CONTROL_ACTIVITY_AWARE_TIMING=${HFD_CONTROL_ACTIVITY_AWARE_TIMING:-1}
|
| 280 |
+
FILTER_KNOWN_TTS_TRANSLATION_MISMATCHES=${HFD_FILTER_KNOWN_TTS_TRANSLATION_MISMATCHES:-1}
|
| 281 |
+
TURN_TAKING_VARIANTS=${HFD_TURN_TAKING_VARIANTS:-4}
|
| 282 |
+
COMMAND_INTERRUPTION_VARIANTS=${HFD_COMMAND_INTERRUPTION_VARIANTS:-1}
|
| 283 |
+
EMERGENCY_STOP_VARIANTS=${HFD_EMERGENCY_STOP_VARIANTS:-2}
|
| 284 |
+
SAFE_HOLD_VARIANTS=${HFD_SAFE_HOLD_VARIANTS:-0}
|
| 285 |
+
BACKCHANNEL_VARIANTS=${HFD_BACKCHANNEL_VARIANTS:-2}
|
| 286 |
+
if [[ "$FULL_DUPLEX_SEMANTICS_VERSION" == "23" \
|
| 287 |
+
|| "$FULL_DUPLEX_SEMANTICS_VERSION" == "24" ]]; then
|
| 288 |
+
HESITATION_VARIANTS=${HFD_HESITATION_VARIANTS:-3}
|
| 289 |
+
PAUSE_VARIANTS=${HFD_PAUSE_VARIANTS:-2}
|
| 290 |
+
HESITATION_TASK_WINDOW_RATIO=${HFD_HESITATION_TASK_WINDOW_RATIO:-0.12}
|
| 291 |
+
PAUSE_TASK_WINDOW_RATIO=${HFD_PAUSE_TASK_WINDOW_RATIO:-0.08}
|
| 292 |
+
else
|
| 293 |
+
HESITATION_VARIANTS=${HFD_HESITATION_VARIANTS:-0}
|
| 294 |
+
PAUSE_VARIANTS=${HFD_PAUSE_VARIANTS:-0}
|
| 295 |
+
HESITATION_TASK_WINDOW_RATIO=${HFD_HESITATION_TASK_WINDOW_RATIO:-0.0}
|
| 296 |
+
PAUSE_TASK_WINDOW_RATIO=${HFD_PAUSE_TASK_WINDOW_RATIO:-0.0}
|
| 297 |
+
fi
|
| 298 |
+
INTERRUPTION_TASK_SESSION_RATIO=${HFD_INTERRUPTION_TASK_SESSION_RATIO:-0.15}
|
| 299 |
+
BACKCHANNEL_TASK_SESSION_RATIO=${HFD_BACKCHANNEL_TASK_SESSION_RATIO:-0.15}
|
| 300 |
+
else
|
| 301 |
+
CONTROL_MIN_START_SECONDS=${HFD_CONTROL_MIN_START_SECONDS:-2.0}
|
| 302 |
+
CONTROL_MAX_START_SECONDS=${HFD_CONTROL_MAX_START_SECONDS:-6.0}
|
| 303 |
+
CONTROL_MIN_REMAINING_SECONDS=${HFD_CONTROL_MIN_REMAINING_SECONDS:-0.4}
|
| 304 |
+
CONTROL_ACTIVITY_AWARE_TIMING=${HFD_CONTROL_ACTIVITY_AWARE_TIMING:-0}
|
| 305 |
+
FILTER_KNOWN_TTS_TRANSLATION_MISMATCHES=${HFD_FILTER_KNOWN_TTS_TRANSLATION_MISMATCHES:-0}
|
| 306 |
+
TURN_TAKING_VARIANTS=${HFD_TURN_TAKING_VARIANTS:-2}
|
| 307 |
+
COMMAND_INTERRUPTION_VARIANTS=${HFD_COMMAND_INTERRUPTION_VARIANTS:-2}
|
| 308 |
+
EMERGENCY_STOP_VARIANTS=${HFD_EMERGENCY_STOP_VARIANTS:-1}
|
| 309 |
+
SAFE_HOLD_VARIANTS=${HFD_SAFE_HOLD_VARIANTS:-1}
|
| 310 |
+
BACKCHANNEL_VARIANTS=${HFD_BACKCHANNEL_VARIANTS:-0}
|
| 311 |
+
HESITATION_VARIANTS=${HFD_HESITATION_VARIANTS:-0}
|
| 312 |
+
PAUSE_VARIANTS=${HFD_PAUSE_VARIANTS:-0}
|
| 313 |
+
HESITATION_TASK_WINDOW_RATIO=${HFD_HESITATION_TASK_WINDOW_RATIO:-0.0}
|
| 314 |
+
PAUSE_TASK_WINDOW_RATIO=${HFD_PAUSE_TASK_WINDOW_RATIO:-0.0}
|
| 315 |
+
INTERRUPTION_TASK_SESSION_RATIO=${HFD_INTERRUPTION_TASK_SESSION_RATIO:-0.0}
|
| 316 |
+
BACKCHANNEL_TASK_SESSION_RATIO=${HFD_BACKCHANNEL_TASK_SESSION_RATIO:-0.0}
|
| 317 |
+
fi
|
| 318 |
+
if [[ "$CONTROL_ACTIVITY_AWARE_TIMING" == "1" ]]; then
|
| 319 |
+
CONTROL_ACTIVITY_TIMING_ARG=--data.control-activity-aware-timing
|
| 320 |
+
elif [[ "$CONTROL_ACTIVITY_AWARE_TIMING" == "0" ]]; then
|
| 321 |
+
CONTROL_ACTIVITY_TIMING_ARG=--data.no-control-activity-aware-timing
|
| 322 |
+
else
|
| 323 |
+
echo "[error] HFD_CONTROL_ACTIVITY_AWARE_TIMING must be 0 or 1." >&2
|
| 324 |
+
exit 2
|
| 325 |
+
fi
|
| 326 |
+
if [[ "$FILTER_KNOWN_TTS_TRANSLATION_MISMATCHES" == "1" ]]; then
|
| 327 |
+
TRANSLATION_FILTER_DATA_ARG=--data.filter-known-tts-translation-mismatches
|
| 328 |
+
elif [[ "$FILTER_KNOWN_TTS_TRANSLATION_MISMATCHES" == "0" ]]; then
|
| 329 |
+
TRANSLATION_FILTER_DATA_ARG=--data.no-filter-known-tts-translation-mismatches
|
| 330 |
+
else
|
| 331 |
+
echo "[error] HFD_FILTER_KNOWN_TTS_TRANSLATION_MISMATCHES must be 0 or 1." >&2
|
| 332 |
+
exit 2
|
| 333 |
+
fi
|
| 334 |
+
CANONICAL_MOTION_DATA_ARG=
|
| 335 |
+
if [[ -n "$CANONICAL_MOTION_MANIFEST" ]]; then
|
| 336 |
+
CANONICAL_MOTION_DATA_ARG=--data.canonical-motion-manifest=$CANONICAL_MOTION_MANIFEST
|
| 337 |
+
fi
|
| 338 |
+
if [[ "$FULL_DUPLEX_SEMANTICS_VERSION" == "21" \
|
| 339 |
+
|| "$FULL_DUPLEX_SEMANTICS_VERSION" == "22" \
|
| 340 |
+
|| "$FULL_DUPLEX_SEMANTICS_VERSION" == "23" \
|
| 341 |
+
|| "$FULL_DUPLEX_SEMANTICS_VERSION" == "24" ]]; then
|
| 342 |
+
if [[ "$FULL_DUPLEX_SEMANTICS_VERSION" == "22" \
|
| 343 |
+
|| "$FULL_DUPLEX_SEMANTICS_VERSION" == "23" \
|
| 344 |
+
|| "$FULL_DUPLEX_SEMANTICS_VERSION" == "24" ]]; then
|
| 345 |
+
CONTROL_TASK_SESSION_RATIO=${HFD_CONTROL_TASK_SESSION_RATIO:-0.10}
|
| 346 |
+
else
|
| 347 |
+
CONTROL_TASK_SESSION_RATIO=${HFD_CONTROL_TASK_SESSION_RATIO:-0.07}
|
| 348 |
+
fi
|
| 349 |
+
CONTROL_FOLLOWUP_RATIO=${HFD_CONTROL_FOLLOWUP_RATIO:-0.50}
|
| 350 |
+
CONTROL_POST_IDLE_MIN_STEPS=${HFD_CONTROL_POST_IDLE_MIN_STEPS:-3}
|
| 351 |
+
if [[ "$FULL_DUPLEX_SEMANTICS_VERSION" == "22" \
|
| 352 |
+
|| "$FULL_DUPLEX_SEMANTICS_VERSION" == "23" \
|
| 353 |
+
|| "$FULL_DUPLEX_SEMANTICS_VERSION" == "24" ]]; then
|
| 354 |
+
CONTROL_POST_IDLE_MAX_STEPS=${HFD_CONTROL_POST_IDLE_MAX_STEPS:-20}
|
| 355 |
+
CONTROL_POST_IDLE_SAMPLING=${HFD_CONTROL_POST_IDLE_SAMPLING:-short_heavy_long_tail}
|
| 356 |
+
STOP_ENTER_LOSS_WEIGHT=${HFD_STOP_ENTER_LOSS_WEIGHT:-0.5}
|
| 357 |
+
STOP_STAY_LOSS_WEIGHT=${HFD_STOP_STAY_LOSS_WEIGHT:-1.0}
|
| 358 |
+
STOP_RELEASE_LOSS_WEIGHT=${HFD_STOP_RELEASE_LOSS_WEIGHT:-1.0}
|
| 359 |
+
else
|
| 360 |
+
CONTROL_POST_IDLE_MAX_STEPS=${HFD_CONTROL_POST_IDLE_MAX_STEPS:-0}
|
| 361 |
+
CONTROL_POST_IDLE_SAMPLING=${HFD_CONTROL_POST_IDLE_SAMPLING:-legacy_uniform}
|
| 362 |
+
STOP_ENTER_LOSS_WEIGHT=${HFD_STOP_ENTER_LOSS_WEIGHT:-0.0}
|
| 363 |
+
STOP_STAY_LOSS_WEIGHT=${HFD_STOP_STAY_LOSS_WEIGHT:-0.0}
|
| 364 |
+
STOP_RELEASE_LOSS_WEIGHT=${HFD_STOP_RELEASE_LOSS_WEIGHT:-0.0}
|
| 365 |
+
fi
|
| 366 |
+
else
|
| 367 |
+
CONTROL_TASK_SESSION_RATIO=${HFD_CONTROL_TASK_SESSION_RATIO:-0.0}
|
| 368 |
+
CONTROL_FOLLOWUP_RATIO=${HFD_CONTROL_FOLLOWUP_RATIO:-0.0}
|
| 369 |
+
CONTROL_POST_IDLE_MIN_STEPS=${HFD_CONTROL_POST_IDLE_MIN_STEPS:-1}
|
| 370 |
+
CONTROL_POST_IDLE_MAX_STEPS=${HFD_CONTROL_POST_IDLE_MAX_STEPS:-0}
|
| 371 |
+
CONTROL_POST_IDLE_SAMPLING=${HFD_CONTROL_POST_IDLE_SAMPLING:-legacy_uniform}
|
| 372 |
+
STOP_ENTER_LOSS_WEIGHT=${HFD_STOP_ENTER_LOSS_WEIGHT:-0.0}
|
| 373 |
+
STOP_STAY_LOSS_WEIGHT=${HFD_STOP_STAY_LOSS_WEIGHT:-0.0}
|
| 374 |
+
STOP_RELEASE_LOSS_WEIGHT=${HFD_STOP_RELEASE_LOSS_WEIGHT:-0.0}
|
| 375 |
+
fi
|
| 376 |
+
CONTROL_COMMAND_MANIFEST=${HFD_CONTROL_COMMAND_MANIFEST:-/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/videogen/hfd_control_commands_v1/manifest.jsonl}
|
| 377 |
+
BACKCHANNEL_COMMAND_MANIFEST=${HFD_BACKCHANNEL_COMMAND_MANIFEST:-/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/videogen/hfd_backchannel_commands_v2_short/manifest.jsonl}
|
| 378 |
+
if [[ "$FULL_DUPLEX_SEMANTICS_VERSION" == "24" ]]; then
|
| 379 |
+
DEFAULT_HESITATION_PAUSE_MANIFEST=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/videogen/hfd_hesitation_pause_v2/manifest.jsonl
|
| 380 |
+
SELF_REPAIR_IMPLICIT_RATIO=${HFD_SELF_REPAIR_IMPLICIT_RATIO:-0.35}
|
| 381 |
+
STATIONARY_IDLE_RESIDUAL_DATA_ARG=--data.stationary-idle-residual-retargeting
|
| 382 |
+
else
|
| 383 |
+
DEFAULT_HESITATION_PAUSE_MANIFEST=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/videogen/hfd_hesitation_pause_v1/manifest.jsonl
|
| 384 |
+
SELF_REPAIR_IMPLICIT_RATIO=${HFD_SELF_REPAIR_IMPLICIT_RATIO:-0.0}
|
| 385 |
+
STATIONARY_IDLE_RESIDUAL_DATA_ARG=--data.no-stationary-idle-residual-retargeting
|
| 386 |
+
fi
|
| 387 |
+
HESITATION_PAUSE_MANIFEST=${HFD_HESITATION_PAUSE_MANIFEST:-$DEFAULT_HESITATION_PAUSE_MANIFEST}
|
| 388 |
+
HESITATION_PAUSE_PRODUCTION_MARKER=${HFD_HESITATION_PAUSE_PRODUCTION_MARKER:-${HESITATION_PAUSE_MANIFEST%/*}/reports/production_complete.json}
|
| 389 |
+
NEW_TASK_MAX_ROWS_PER_INSTRUCTION_TRAIN=${HFD_NEW_TASK_MAX_ROWS_PER_INSTRUCTION_TRAIN:-32}
|
| 390 |
+
NEW_TASK_MAX_ROWS_PER_INSTRUCTION_VAL=${HFD_NEW_TASK_MAX_ROWS_PER_INSTRUCTION_VAL:-8}
|
| 391 |
+
HESITATION_FILLED_RATIO=${HFD_HESITATION_FILLED_RATIO:-0.60}
|
| 392 |
+
BACKCHANNEL_MIN_START_SECONDS=${HFD_BACKCHANNEL_MIN_START_SECONDS:-1.2}
|
| 393 |
+
BACKCHANNEL_MAX_START_SECONDS=${HFD_BACKCHANNEL_MAX_START_SECONDS:-6.0}
|
| 394 |
+
BACKCHANNEL_MIN_REMAINING_SECONDS=${HFD_BACKCHANNEL_MIN_REMAINING_SECONDS:-1.2}
|
| 395 |
+
SAFE_HOLD_STABILITY_WINDOW_FRAMES=${HFD_SAFE_HOLD_STABILITY_WINDOW_FRAMES:-20}
|
| 396 |
+
if [[ "$FULL_DUPLEX_SEMANTICS_VERSION" == "21" \
|
| 397 |
+
|| "$FULL_DUPLEX_SEMANTICS_VERSION" == "22" \
|
| 398 |
+
|| "$FULL_DUPLEX_SEMANTICS_VERSION" == "23" \
|
| 399 |
+
|| "$FULL_DUPLEX_SEMANTICS_VERSION" == "24" ]]; then
|
| 400 |
+
INDEPENDENT_TASK_POOLS_ARG=--data.independent-task-pools
|
| 401 |
+
else
|
| 402 |
+
INDEPENDENT_TASK_POOLS_ARG=--data.no-independent-task-pools
|
| 403 |
+
fi
|
| 404 |
+
if [[ "$FULL_DUPLEX_SEMANTICS_VERSION" == "13" \
|
| 405 |
+
|| "$FULL_DUPLEX_SEMANTICS_VERSION" == "14" \
|
| 406 |
+
|| "$FULL_DUPLEX_SEMANTICS_VERSION" == "15" \
|
| 407 |
+
|| "$FULL_DUPLEX_SEMANTICS_VERSION" == "16" \
|
| 408 |
+
|| "$FULL_DUPLEX_SEMANTICS_VERSION" == "17" \
|
| 409 |
+
|| "$FULL_DUPLEX_SEMANTICS_VERSION" == "18" \
|
| 410 |
+
|| "$FULL_DUPLEX_SEMANTICS_VERSION" == "19" \
|
| 411 |
+
|| "$FULL_DUPLEX_SEMANTICS_VERSION" == "20" \
|
| 412 |
+
|| "$FULL_DUPLEX_SEMANTICS_VERSION" == "21" \
|
| 413 |
+
|| "$FULL_DUPLEX_SEMANTICS_VERSION" == "22" \
|
| 414 |
+
|| "$FULL_DUPLEX_SEMANTICS_VERSION" == "23" \
|
| 415 |
+
|| "$FULL_DUPLEX_SEMANTICS_VERSION" == "24" ]]; then
|
| 416 |
+
INITIAL_IDLE_MAX_STEPS=${HFD_INITIAL_IDLE_MAX_STEPS:-7}
|
| 417 |
+
MULTI_TURN_DATA_ARG=--data.multi-turn-sessions
|
| 418 |
+
if [[ "$FULL_DUPLEX_SEMANTICS_VERSION" == "14" \
|
| 419 |
+
|| "$FULL_DUPLEX_SEMANTICS_VERSION" == "15" \
|
| 420 |
+
|| "$FULL_DUPLEX_SEMANTICS_VERSION" == "16" \
|
| 421 |
+
|| "$FULL_DUPLEX_SEMANTICS_VERSION" == "17" \
|
| 422 |
+
|| "$FULL_DUPLEX_SEMANTICS_VERSION" == "18" \
|
| 423 |
+
|| "$FULL_DUPLEX_SEMANTICS_VERSION" == "19" \
|
| 424 |
+
|| "$FULL_DUPLEX_SEMANTICS_VERSION" == "20" \
|
| 425 |
+
|| "$FULL_DUPLEX_SEMANTICS_VERSION" == "21" \
|
| 426 |
+
|| "$FULL_DUPLEX_SEMANTICS_VERSION" == "22" \
|
| 427 |
+
|| "$FULL_DUPLEX_SEMANTICS_VERSION" == "23" \
|
| 428 |
+
|| "$FULL_DUPLEX_SEMANTICS_VERSION" == "24" ]]; then
|
| 429 |
+
CAUSAL_LISTEN_DATA_ARG=--data.causal-listen-timing
|
| 430 |
+
else
|
| 431 |
+
CAUSAL_LISTEN_DATA_ARG=--data.no-causal-listen-timing
|
| 432 |
+
fi
|
| 433 |
+
else
|
| 434 |
+
INITIAL_IDLE_MAX_STEPS=${HFD_INITIAL_IDLE_MAX_STEPS:-5}
|
| 435 |
+
MULTI_TURN_DATA_ARG=--data.no-multi-turn-sessions
|
| 436 |
+
CAUSAL_LISTEN_DATA_ARG=--data.no-causal-listen-timing
|
| 437 |
+
fi
|
| 438 |
+
if [[ "$FULL_DUPLEX_SEMANTICS_VERSION" == "20" \
|
| 439 |
+
|| "$FULL_DUPLEX_SEMANTICS_VERSION" == "21" \
|
| 440 |
+
|| "$FULL_DUPLEX_SEMANTICS_VERSION" == "22" \
|
| 441 |
+
|| "$FULL_DUPLEX_SEMANTICS_VERSION" == "23" \
|
| 442 |
+
|| "$FULL_DUPLEX_SEMANTICS_VERSION" == "24" ]]; then
|
| 443 |
+
INTERRUPTION_DATA_ARG=--data.interruption-sessions
|
| 444 |
+
else
|
| 445 |
+
INTERRUPTION_DATA_ARG=--data.no-interruption-sessions
|
| 446 |
+
fi
|
| 447 |
+
if [[ "$FULL_DUPLEX_SEMANTICS_VERSION" == "15" \
|
| 448 |
+
|| "$FULL_DUPLEX_SEMANTICS_VERSION" == "16" \
|
| 449 |
+
|| "$FULL_DUPLEX_SEMANTICS_VERSION" == "17" \
|
| 450 |
+
|| "$FULL_DUPLEX_SEMANTICS_VERSION" == "18" \
|
| 451 |
+
|| "$FULL_DUPLEX_SEMANTICS_VERSION" == "19" \
|
| 452 |
+
|| "$FULL_DUPLEX_SEMANTICS_VERSION" == "20" \
|
| 453 |
+
|| "$FULL_DUPLEX_SEMANTICS_VERSION" == "21" \
|
| 454 |
+
|| "$FULL_DUPLEX_SEMANTICS_VERSION" == "22" \
|
| 455 |
+
|| "$FULL_DUPLEX_SEMANTICS_VERSION" == "23" \
|
| 456 |
+
|| "$FULL_DUPLEX_SEMANTICS_VERSION" == "24" ]]; then
|
| 457 |
+
STATIONARY_HEADING_DATA_ARG=--data.stationary-heading-alignment
|
| 458 |
+
else
|
| 459 |
+
STATIONARY_HEADING_DATA_ARG=--data.no-stationary-heading-alignment
|
| 460 |
+
fi
|
| 461 |
+
if [[ "$FULL_DUPLEX_SEMANTICS_VERSION" == "18" \
|
| 462 |
+
|| "$FULL_DUPLEX_SEMANTICS_VERSION" == "19" \
|
| 463 |
+
|| "$FULL_DUPLEX_SEMANTICS_VERSION" == "20" \
|
| 464 |
+
|| "$FULL_DUPLEX_SEMANTICS_VERSION" == "21" \
|
| 465 |
+
|| "$FULL_DUPLEX_SEMANTICS_VERSION" == "22" \
|
| 466 |
+
|| "$FULL_DUPLEX_SEMANTICS_VERSION" == "23" \
|
| 467 |
+
|| "$FULL_DUPLEX_SEMANTICS_VERSION" == "24" ]]; then
|
| 468 |
+
# Reserve enough native Qwen context for the 18-token readout query, the
|
| 469 |
+
# audited <=33-token transcript, and its EOS without truncating production KV.
|
| 470 |
+
MAX_STREAM_STEPS=${HFD_MAX_STREAM_STEPS:-1919}
|
| 471 |
+
else
|
| 472 |
+
MAX_STREAM_STEPS=${HFD_MAX_STREAM_STEPS:-1923}
|
| 473 |
+
fi
|
| 474 |
+
MOTION_CACHE_SIZE=${HFD_MOTION_CACHE_SIZE:-256}
|
| 475 |
+
MOTION_METADATA_FPS=${HFD_MOTION_METADATA_FPS:-120}
|
| 476 |
+
MAX_VOICE_WINDOWS=${HFD_MAX_VOICE_WINDOWS:-30}
|
| 477 |
+
MAX_VOICE_MEDIAN_RATIO=${HFD_MAX_VOICE_MEDIAN_RATIO:-3.0}
|
| 478 |
+
if [[ "$FULL_DUPLEX_SEMANTICS_VERSION" == "22" \
|
| 479 |
+
|| "$FULL_DUPLEX_SEMANTICS_VERSION" == "23" \
|
| 480 |
+
|| "$FULL_DUPLEX_SEMANTICS_VERSION" == "24" ]]; then
|
| 481 |
+
SAFE_STATIONARY_DATA_ARG=--data.safe-stationary-anchors
|
| 482 |
+
DYNAMIC_TASK_ENTRY_DATA_ARG=--data.dynamic-task-entry-blend
|
| 483 |
+
if [[ "$EXPERIMENT_MODE" == "formal" ]]; then
|
| 484 |
+
LONG_SESSION_DROP_RATIO=${HFD_LONG_SESSION_DROP_RATIO:-0.001}
|
| 485 |
+
else
|
| 486 |
+
LONG_SESSION_DROP_RATIO=${HFD_LONG_SESSION_DROP_RATIO:-0.0}
|
| 487 |
+
fi
|
| 488 |
+
else
|
| 489 |
+
SAFE_STATIONARY_DATA_ARG=--data.no-safe-stationary-anchors
|
| 490 |
+
DYNAMIC_TASK_ENTRY_DATA_ARG=--data.no-dynamic-task-entry-blend
|
| 491 |
+
LONG_SESSION_DROP_RATIO=${HFD_LONG_SESSION_DROP_RATIO:-0.0}
|
| 492 |
+
fi
|
| 493 |
+
# Recyclable source slots keep almost every optimizer step full. A controlled
|
| 494 |
+
# four-GPU A/B found that the former 1e-4 peak over-updated this denser stream;
|
| 495 |
+
# 8e-5 improved held-out action loss without changing data, KV, or supervision.
|
| 496 |
+
# Existing full-state resumes restore their checkpoint optimizer/scheduler.
|
| 497 |
+
LEARNING_RATE=${HFD_LEARNING_RATE:-8e-5}
|
| 498 |
+
RUN_TIMESTAMP=${HFD_RUN_TIMESTAMP:-$(date +'%y%m%d%H%M')}
|
| 499 |
+
RESUME_FROM_CHECKPOINT=${HFD_RESUME_FROM_CHECKPOINT:-None}
|
| 500 |
+
RESUME_LOAD_FULL_STATE=${HFD_RESUME_LOAD_FULL_STATE:-0}
|
| 501 |
+
ALLOW_V11_WEIGHTS_INIT=${HFD_ALLOW_V11_WEIGHTS_INIT:-0}
|
| 502 |
+
export HFD_ALLOW_V11_WEIGHTS_INIT=$ALLOW_V11_WEIGHTS_INIT
|
| 503 |
+
export HFD_WANDB_VERBOSE_METRICS=${HFD_WANDB_VERBOSE_METRICS:-0}
|
| 504 |
+
WANDB_RESUME_MODE=${HFD_WANDB_RESUME:-never}
|
| 505 |
+
WANDB_RUN_ID=${HFD_WANDB_ID:-}
|
| 506 |
+
|
| 507 |
+
RESUME_RUN_CONFIG=
|
| 508 |
+
if [[ "$RESUME_LOAD_FULL_STATE" == "1" && "$RESUME_FROM_CHECKPOINT" != "None" ]]; then
|
| 509 |
+
RESUME_RUN_CONFIG=$(
|
| 510 |
+
python - "$RESUME_FROM_CHECKPOINT" <<'PY'
|
| 511 |
+
import sys
|
| 512 |
+
from pathlib import Path
|
| 513 |
+
|
| 514 |
+
checkpoint = Path(sys.argv[1]).expanduser().resolve()
|
| 515 |
+
for candidate in (checkpoint / "run_config.json", checkpoint.parent / "run_config.json"):
|
| 516 |
+
if candidate.is_file():
|
| 517 |
+
print(candidate)
|
| 518 |
+
break
|
| 519 |
+
PY
|
| 520 |
+
)
|
| 521 |
+
fi
|
| 522 |
+
if [[ "$FULL_DUPLEX_SEMANTICS_VERSION" == "23" \
|
| 523 |
+
|| "$FULL_DUPLEX_SEMANTICS_VERSION" == "24" ]]; then
|
| 524 |
+
if (( HESITATION_VARIANTS < 1 || PAUSE_VARIANTS < 1 )); then
|
| 525 |
+
echo "[error] Semantics 23/24 requires positive hesitation and pause variants." >&2
|
| 526 |
+
exit 2
|
| 527 |
+
fi
|
| 528 |
+
if ! [[ "$NEW_TASK_MAX_ROWS_PER_INSTRUCTION_TRAIN" =~ ^[1-9][0-9]*$ \
|
| 529 |
+
&& "$NEW_TASK_MAX_ROWS_PER_INSTRUCTION_VAL" =~ ^[1-9][0-9]*$ ]]; then
|
| 530 |
+
echo "[error] New-task per-instruction row caps must be positive integers." >&2
|
| 531 |
+
exit 2
|
| 532 |
+
fi
|
| 533 |
+
elif (( HESITATION_VARIANTS != 0 || PAUSE_VARIANTS != 0 )) \
|
| 534 |
+
|| [[ "$HESITATION_TASK_WINDOW_RATIO" != "0" \
|
| 535 |
+
&& "$HESITATION_TASK_WINDOW_RATIO" != "0.0" ]] \
|
| 536 |
+
|| [[ "$PAUSE_TASK_WINDOW_RATIO" != "0" \
|
| 537 |
+
&& "$PAUSE_TASK_WINDOW_RATIO" != "0.0" ]]; then
|
| 538 |
+
echo "[error] Hesitation/pause pools are semantics-23/24 only." >&2
|
| 539 |
+
exit 2
|
| 540 |
+
fi
|
| 541 |
+
|
| 542 |
+
# A legacy full-state run used fixed source slots. Preserve that scheduler by
|
| 543 |
+
# default on resume; an explicit environment override starts a deliberate new
|
| 544 |
+
# scheduling regime. Fresh runs use recyclable slots.
|
| 545 |
+
if [[ -z "$DYNAMIC_SOURCE_REFILL_EXPLICIT" && -n "$RESUME_RUN_CONFIG" ]]; then
|
| 546 |
+
DYNAMIC_SOURCE_REFILL=$(
|
| 547 |
+
python - "$RESUME_RUN_CONFIG" <<'PY'
|
| 548 |
+
import json
|
| 549 |
+
import sys
|
| 550 |
+
|
| 551 |
+
with open(sys.argv[1], "r", encoding="utf-8") as handle:
|
| 552 |
+
config = json.load(handle)
|
| 553 |
+
print(1 if config.get("train", {}).get("dynamic_source_refill", False) else 0)
|
| 554 |
+
PY
|
| 555 |
+
)
|
| 556 |
+
fi
|
| 557 |
+
|
| 558 |
+
# Preserve the historical optimizer parameter-group layout for a full-state
|
| 559 |
+
# resume unless the caller explicitly opts into a new control-head LR.
|
| 560 |
+
if [[ -z "$CONTROL_HEAD_LR_MULTIPLIER_EXPLICIT" \
|
| 561 |
+
&& -n "$RESUME_RUN_CONFIG" ]]; then
|
| 562 |
+
CONTROL_HEAD_LR_MULTIPLIER=$(
|
| 563 |
+
python - "$RESUME_RUN_CONFIG" <<'PY'
|
| 564 |
+
import json
|
| 565 |
+
import sys
|
| 566 |
+
|
| 567 |
+
with open(sys.argv[1], "r", encoding="utf-8") as handle:
|
| 568 |
+
config = json.load(handle)
|
| 569 |
+
print(float(config.get("train", {}).get("control_head_lr_multiplier", 1.0)))
|
| 570 |
+
PY
|
| 571 |
+
)
|
| 572 |
+
fi
|
| 573 |
+
|
| 574 |
+
# Fresh one-million-step runs warm the two newly initialized action/control
|
| 575 |
+
# groups up over the first 1%. A full-state resume inherits the original
|
| 576 |
+
# run's value unless the caller explicitly overrides HFD_WARMUP_STEPS, so an
|
| 577 |
+
# older zero-warmup scheduler can still continue without a silent LR change.
|
| 578 |
+
if [[ -n "${HFD_WARMUP_STEPS+x}" ]]; then
|
| 579 |
+
WARMUP_STEPS=$HFD_WARMUP_STEPS
|
| 580 |
+
else
|
| 581 |
+
if [[ "$FULL_DUPLEX_SEMANTICS_VERSION" == "23" \
|
| 582 |
+
|| "$FULL_DUPLEX_SEMANTICS_VERSION" == "24" ]]; then
|
| 583 |
+
WARMUP_STEPS=12500
|
| 584 |
+
else
|
| 585 |
+
WARMUP_STEPS=10000
|
| 586 |
+
fi
|
| 587 |
+
if [[ -n "$RESUME_RUN_CONFIG" ]]; then
|
| 588 |
+
WARMUP_STEPS=$(
|
| 589 |
+
python - "$RESUME_RUN_CONFIG" <<'PY'
|
| 590 |
+
import json
|
| 591 |
+
import sys
|
| 592 |
+
|
| 593 |
+
with open(sys.argv[1], "r", encoding="utf-8") as handle:
|
| 594 |
+
config = json.load(handle)
|
| 595 |
+
print(int(config.get("train", {}).get("warmup_steps", 0) or 0))
|
| 596 |
+
PY
|
| 597 |
+
)
|
| 598 |
+
fi
|
| 599 |
+
fi
|
| 600 |
+
|
| 601 |
+
for value_name in \
|
| 602 |
+
TRAIN_BATCH_SIZE \
|
| 603 |
+
EFFECTIVE_SOURCES_PER_GPU \
|
| 604 |
+
QWEN_KV_GRADIENT_HORIZON \
|
| 605 |
+
QWEN_TRAINABLE_LAST_LAYERS \
|
| 606 |
+
DENSE_WINDOW_ACCUMULATION_STEPS \
|
| 607 |
+
MOTION_DURATION_DISTRIBUTION_BINS; do
|
| 608 |
+
value=${!value_name}
|
| 609 |
+
if [[ ! "$value" =~ ^[1-9][0-9]*$ ]]; then
|
| 610 |
+
echo "[error] $value_name must be a positive integer, got $value" >&2
|
| 611 |
+
exit 2
|
| 612 |
+
fi
|
| 613 |
+
done
|
| 614 |
+
if [[ "$CONTROL_CONDITION_TRAIN_MODE" != "hard" \
|
| 615 |
+
&& "$CONTROL_CONDITION_TRAIN_MODE" != "straight_through" \
|
| 616 |
+
&& "$CONTROL_CONDITION_TRAIN_MODE" != "soft" ]]; then
|
| 617 |
+
echo "[error] HFD_CONTROL_CONDITION_TRAIN_MODE must be hard, straight_through or soft." >&2
|
| 618 |
+
exit 2
|
| 619 |
+
fi
|
| 620 |
+
if [[ "$CONTROL_CONDITION_INFERENCE_MODE" != "hard" \
|
| 621 |
+
&& "$CONTROL_CONDITION_INFERENCE_MODE" != "soft" ]]; then
|
| 622 |
+
echo "[error] HFD_CONTROL_CONDITION_INFERENCE_MODE must be hard or soft." >&2
|
| 623 |
+
exit 2
|
| 624 |
+
fi
|
| 625 |
+
SOURCE_ACCUMULATION_GROUPS=${SOURCE_ACCUMULATION_GROUPS:-1}
|
| 626 |
+
if (( SOURCE_ACCUMULATION_GROUPS != 1 \
|
| 627 |
+
|| EFFECTIVE_SOURCES_PER_GPU != TRAIN_BATCH_SIZE )); then
|
| 628 |
+
echo "[error] Window-step training updates every clip and requires source accumulation=1." >&2
|
| 629 |
+
echo " effective sources/GPU must equal train batch: $EFFECTIVE_SOURCES_PER_GPU != $TRAIN_BATCH_SIZE" >&2
|
| 630 |
+
exit 2
|
| 631 |
+
fi
|
| 632 |
+
case "$EXPERIMENT_MODE" in
|
| 633 |
+
formal)
|
| 634 |
+
;;
|
| 635 |
+
overfit50)
|
| 636 |
+
if (( LIMIT_SOURCES != 50 )); then
|
| 637 |
+
echo "[error] HFDv2 overfit50 requires HFD_LIMIT_SOURCES=50." >&2
|
| 638 |
+
exit 2
|
| 639 |
+
fi
|
| 640 |
+
if (( SOURCE_ACCUMULATION_GROUPS != 1 || EFFECTIVE_SOURCES_PER_GPU != TRAIN_BATCH_SIZE )); then
|
| 641 |
+
echo "[error] overfit50 also requires one optimizer update per aligned window." >&2
|
| 642 |
+
exit 2
|
| 643 |
+
fi
|
| 644 |
+
;;
|
| 645 |
+
*)
|
| 646 |
+
echo "[error] Unsupported HFD_EXPERIMENT_MODE=$EXPERIMENT_MODE (expected formal or overfit50)." >&2
|
| 647 |
+
exit 2
|
| 648 |
+
;;
|
| 649 |
+
esac
|
| 650 |
+
if [[ "$GRADIENT_ACCUMULATION_STEPS" != "1" \
|
| 651 |
+
|| "$TBPTT_UNROLL_STEPS" != "0" \
|
| 652 |
+
|| "$QWEN_KV_GRADIENT_HORIZON" != "1" ]]; then
|
| 653 |
+
echo "[error] Window-step training requires gradient accumulation=1, TBPTT=0 and Qwen horizon=1." >&2
|
| 654 |
+
exit 2
|
| 655 |
+
fi
|
| 656 |
+
if (( MAX_STREAM_STEPS > 1923 )); then
|
| 657 |
+
echo "[error] Qwen-safe max_stream_steps must be <=1923." >&2
|
| 658 |
+
exit 2
|
| 659 |
+
fi
|
| 660 |
+
if [[ "$FULL_DUPLEX_SEMANTICS_VERSION" == "18" \
|
| 661 |
+
|| "$FULL_DUPLEX_SEMANTICS_VERSION" == "19" \
|
| 662 |
+
|| "$FULL_DUPLEX_SEMANTICS_VERSION" == "20" \
|
| 663 |
+
|| "$FULL_DUPLEX_SEMANTICS_VERSION" == "21" \
|
| 664 |
+
|| "$FULL_DUPLEX_SEMANTICS_VERSION" == "22" \
|
| 665 |
+
|| "$FULL_DUPLEX_SEMANTICS_VERSION" == "23" \
|
| 666 |
+
|| "$FULL_DUPLEX_SEMANTICS_VERSION" == "24" ]] \
|
| 667 |
+
&& (( MAX_STREAM_STEPS > 1919 )); then
|
| 668 |
+
echo "[error] Semantics 18-24 reserves text-readout context and requires max_stream_steps<=1919." >&2
|
| 669 |
+
exit 2
|
| 670 |
+
fi
|
| 671 |
+
if [[ "$FULL_DUPLEX_SEMANTICS_VERSION" == "13" \
|
| 672 |
+
|| "$FULL_DUPLEX_SEMANTICS_VERSION" == "14" \
|
| 673 |
+
|| "$FULL_DUPLEX_SEMANTICS_VERSION" == "15" \
|
| 674 |
+
|| "$FULL_DUPLEX_SEMANTICS_VERSION" == "16" \
|
| 675 |
+
|| "$FULL_DUPLEX_SEMANTICS_VERSION" == "17" \
|
| 676 |
+
|| "$FULL_DUPLEX_SEMANTICS_VERSION" == "18" \
|
| 677 |
+
|| "$FULL_DUPLEX_SEMANTICS_VERSION" == "19" \
|
| 678 |
+
|| "$FULL_DUPLEX_SEMANTICS_VERSION" == "20" \
|
| 679 |
+
|| "$FULL_DUPLEX_SEMANTICS_VERSION" == "21" \
|
| 680 |
+
|| "$FULL_DUPLEX_SEMANTICS_VERSION" == "22" \
|
| 681 |
+
|| "$FULL_DUPLEX_SEMANTICS_VERSION" == "23" \
|
| 682 |
+
|| "$FULL_DUPLEX_SEMANTICS_VERSION" == "24" ]] \
|
| 683 |
+
&& (( INITIAL_IDLE_MAX_STEPS != 7 \
|
| 684 |
+
|| SESSION_MIN_TURNS != 2 \
|
| 685 |
+
|| SESSION_MAX_TURNS != 4 \
|
| 686 |
+
|| TURN_TRANSITION_FRAMES != 50 )); then
|
| 687 |
+
echo "[error] Multi-turn semantics requires 2-4 turns and a 50-frame turn transition." >&2
|
| 688 |
+
exit 2
|
| 689 |
+
fi
|
| 690 |
+
if [[ "$FULL_DUPLEX_SEMANTICS_VERSION" != "13" \
|
| 691 |
+
&& "$FULL_DUPLEX_SEMANTICS_VERSION" != "14" \
|
| 692 |
+
&& "$FULL_DUPLEX_SEMANTICS_VERSION" != "15" \
|
| 693 |
+
&& "$FULL_DUPLEX_SEMANTICS_VERSION" != "16" \
|
| 694 |
+
&& "$FULL_DUPLEX_SEMANTICS_VERSION" != "17" \
|
| 695 |
+
&& "$FULL_DUPLEX_SEMANTICS_VERSION" != "18" \
|
| 696 |
+
&& "$FULL_DUPLEX_SEMANTICS_VERSION" != "19" \
|
| 697 |
+
&& "$FULL_DUPLEX_SEMANTICS_VERSION" != "20" \
|
| 698 |
+
&& "$FULL_DUPLEX_SEMANTICS_VERSION" != "21" \
|
| 699 |
+
&& "$FULL_DUPLEX_SEMANTICS_VERSION" != "22" \
|
| 700 |
+
&& "$FULL_DUPLEX_SEMANTICS_VERSION" != "23" \
|
| 701 |
+
&& "$FULL_DUPLEX_SEMANTICS_VERSION" != "24" ]] \
|
| 702 |
+
&& (( INITIAL_IDLE_MAX_STEPS != 5 )); then
|
| 703 |
+
echo "[error] Historical single-turn semantics require idle_max=5." >&2
|
| 704 |
+
exit 2
|
| 705 |
+
fi
|
| 706 |
+
if [[ "$FULL_DUPLEX_SEMANTICS_VERSION" == "17" \
|
| 707 |
+
|| "$FULL_DUPLEX_SEMANTICS_VERSION" == "18" \
|
| 708 |
+
|| "$FULL_DUPLEX_SEMANTICS_VERSION" == "19" \
|
| 709 |
+
|| "$FULL_DUPLEX_SEMANTICS_VERSION" == "20" \
|
| 710 |
+
|| "$FULL_DUPLEX_SEMANTICS_VERSION" == "21" \
|
| 711 |
+
|| "$FULL_DUPLEX_SEMANTICS_VERSION" == "22" \
|
| 712 |
+
|| "$FULL_DUPLEX_SEMANTICS_VERSION" == "23" \
|
| 713 |
+
|| "$FULL_DUPLEX_SEMANTICS_VERSION" == "24" ]]; then
|
| 714 |
+
if (( SESSION_VOICE_VARIANTS < 2 )); then
|
| 715 |
+
echo "[error] Semantics 17-24 requires HFD_SESSION_VOICE_VARIANTS>=2." >&2
|
| 716 |
+
exit 2
|
| 717 |
+
fi
|
| 718 |
+
elif (( SESSION_VOICE_VARIANTS != 0 )); then
|
| 719 |
+
echo "[error] Historical semantics require HFD_SESSION_VOICE_VARIANTS=0." >&2
|
| 720 |
+
exit 2
|
| 721 |
+
fi
|
| 722 |
+
if [[ "$FULL_DUPLEX_SEMANTICS_VERSION" == "21" \
|
| 723 |
+
|| "$FULL_DUPLEX_SEMANTICS_VERSION" == "22" \
|
| 724 |
+
|| "$FULL_DUPLEX_SEMANTICS_VERSION" == "23" \
|
| 725 |
+
|| "$FULL_DUPLEX_SEMANTICS_VERSION" == "24" ]]; then
|
| 726 |
+
if (( TURN_TAKING_VARIANTS < 1 \
|
| 727 |
+
|| COMMAND_INTERRUPTION_VARIANTS < 0 \
|
| 728 |
+
|| EMERGENCY_STOP_VARIANTS < 0 \
|
| 729 |
+
|| SAFE_HOLD_VARIANTS < 0 \
|
| 730 |
+
|| BACKCHANNEL_VARIANTS < 0 \
|
| 731 |
+
|| HESITATION_VARIANTS < 0 \
|
| 732 |
+
|| PAUSE_VARIANTS < 0 )); then
|
| 733 |
+
echo "[error] Semantics 21-24 task weights must be non-negative and retain turn-taking." >&2
|
| 734 |
+
exit 2
|
| 735 |
+
fi
|
| 736 |
+
if [[ "${HFD_CONFIG_ONLY:-0}" != "1" \
|
| 737 |
+
&& ( EMERGENCY_STOP_VARIANTS > 0 || SAFE_HOLD_VARIANTS > 0 ) \
|
| 738 |
+
&& ! -f "$CONTROL_COMMAND_MANIFEST" ]]; then
|
| 739 |
+
echo "[error] Missing semantics-21 control TTS manifest: $CONTROL_COMMAND_MANIFEST" >&2
|
| 740 |
+
exit 2
|
| 741 |
+
fi
|
| 742 |
+
if [[ "${HFD_CONFIG_ONLY:-0}" != "1" \
|
| 743 |
+
&& BACKCHANNEL_VARIANTS > 0 \
|
| 744 |
+
&& ! -f "$BACKCHANNEL_COMMAND_MANIFEST" ]]; then
|
| 745 |
+
echo "[error] Missing semantics-22/23/24 backchannel TTS manifest: $BACKCHANNEL_COMMAND_MANIFEST" >&2
|
| 746 |
+
exit 2
|
| 747 |
+
fi
|
| 748 |
+
if [[ ( "$FULL_DUPLEX_SEMANTICS_VERSION" == "23" \
|
| 749 |
+
|| "$FULL_DUPLEX_SEMANTICS_VERSION" == "24" ) \
|
| 750 |
+
&& "${HFD_CONFIG_ONLY:-0}" != "1" \
|
| 751 |
+
&& ! -f "$HESITATION_PAUSE_MANIFEST" ]]; then
|
| 752 |
+
echo "[error] Missing semantics-23/24 hesitation/pause TTS manifest: $HESITATION_PAUSE_MANIFEST" >&2
|
| 753 |
+
exit 2
|
| 754 |
+
fi
|
| 755 |
+
if [[ ( "$FULL_DUPLEX_SEMANTICS_VERSION" == "23" \
|
| 756 |
+
|| "$FULL_DUPLEX_SEMANTICS_VERSION" == "24" ) \
|
| 757 |
+
&& "${HFD_CONFIG_ONLY:-0}" != "1" \
|
| 758 |
+
&& ! -f "$HESITATION_PAUSE_PRODUCTION_MARKER" ]]; then
|
| 759 |
+
echo "[error] Missing semantics-23/24 hesitation/pause production marker: $HESITATION_PAUSE_PRODUCTION_MARKER" >&2
|
| 760 |
+
exit 2
|
| 761 |
+
fi
|
| 762 |
+
fi
|
| 763 |
+
if [[ "$FULL_DUPLEX_SEMANTICS_VERSION" == "22" \
|
| 764 |
+
|| "$FULL_DUPLEX_SEMANTICS_VERSION" == "23" \
|
| 765 |
+
|| "$FULL_DUPLEX_SEMANTICS_VERSION" == "24" ]]; then
|
| 766 |
+
if (( SAFE_HOLD_VARIANTS != 0 )); then
|
| 767 |
+
echo "[error] Semantics 22-24 keeps safe-hold code but disables its train pool; set HFD_SAFE_HOLD_VARIANTS=0." >&2
|
| 768 |
+
exit 2
|
| 769 |
+
fi
|
| 770 |
+
if (( MOTION_DURATION_DISTRIBUTION_BINS < 4 )); then
|
| 771 |
+
echo "[error] Semantics 22-24 requires at least four motion-duration bins." >&2
|
| 772 |
+
exit 2
|
| 773 |
+
fi
|
| 774 |
+
python - "$LONG_SESSION_DROP_RATIO" <<'PY'
|
| 775 |
+
import math
|
| 776 |
+
import sys
|
| 777 |
+
|
| 778 |
+
ratio = float(sys.argv[1])
|
| 779 |
+
if not math.isfinite(ratio) or not 0.0 <= ratio < 0.01:
|
| 780 |
+
raise SystemExit(
|
| 781 |
+
"[error] HFD_LONG_SESSION_DROP_RATIO must be within [0, 0.01)"
|
| 782 |
+
)
|
| 783 |
+
PY
|
| 784 |
+
elif (( MOTION_DURATION_DISTRIBUTION_BINS != 1 \
|
| 785 |
+
|| DENSE_WINDOW_ACCUMULATION_STEPS != 1 )) \
|
| 786 |
+
|| [[ "$LONG_SESSION_DROP_RATIO" != "0.0" && "$LONG_SESSION_DROP_RATIO" != "0" ]]; then
|
| 787 |
+
echo "[error] Duration distributions and dense-window accumulation are semantics-22/23/24 only." >&2
|
| 788 |
+
exit 2
|
| 789 |
+
fi
|
| 790 |
+
if [[ "$FULL_DUPLEX_SEMANTICS_VERSION" == "20" ]] \
|
| 791 |
+
&& (( SESSION_VOICE_VARIANTS != 2 )); then
|
| 792 |
+
echo "[error] Semantics 20 requires exactly two session variants for a 1:1 behavior mix." >&2
|
| 793 |
+
exit 2
|
| 794 |
+
fi
|
| 795 |
+
if [[ "$ACTION_RECOMPUTE" != "0" && "$ACTION_RECOMPUTE" != "1" ]]; then
|
| 796 |
+
echo "[error] HFD_ACTION_RECOMPUTE must be 0 or 1, got $ACTION_RECOMPUTE" >&2
|
| 797 |
+
exit 2
|
| 798 |
+
fi
|
| 799 |
+
if [[ "$COMPACT_INACTIVE_ACTION_ROWS" != "0" && "$COMPACT_INACTIVE_ACTION_ROWS" != "1" ]]; then
|
| 800 |
+
echo "[error] HFD_COMPACT_INACTIVE_ACTION_ROWS must be 0 or 1, got $COMPACT_INACTIVE_ACTION_ROWS" >&2
|
| 801 |
+
exit 2
|
| 802 |
+
fi
|
| 803 |
+
if [[ "$DYNAMIC_SOURCE_REFILL" != "0" && "$DYNAMIC_SOURCE_REFILL" != "1" ]]; then
|
| 804 |
+
echo "[error] HFD_DYNAMIC_SOURCE_REFILL must be 0 or 1, got $DYNAMIC_SOURCE_REFILL" >&2
|
| 805 |
+
exit 2
|
| 806 |
+
fi
|
| 807 |
+
if ! [[ "$DYNAMIC_SOURCE_CONTEXT_RESERVE_TOKENS" =~ ^[0-9]+$ ]]; then
|
| 808 |
+
echo "[error] HFD_DYNAMIC_SOURCE_CONTEXT_RESERVE_TOKENS must be a non-negative integer, got $DYNAMIC_SOURCE_CONTEXT_RESERVE_TOKENS" >&2
|
| 809 |
+
exit 2
|
| 810 |
+
fi
|
| 811 |
+
if ! [[ "$WARMUP_STEPS" =~ ^[0-9]+$ ]]; then
|
| 812 |
+
echo "[error] HFD_WARMUP_STEPS must be a non-negative integer, got $WARMUP_STEPS" >&2
|
| 813 |
+
exit 2
|
| 814 |
+
fi
|
| 815 |
+
if [[ "$COMPACT_INACTIVE_ACTION_ROWS" == "1" ]]; then
|
| 816 |
+
COMPACT_INACTIVE_ACTION_ROWS_ARG=--train.compact-inactive-action-rows
|
| 817 |
+
else
|
| 818 |
+
COMPACT_INACTIVE_ACTION_ROWS_ARG=--train.no-compact-inactive-action-rows
|
| 819 |
+
fi
|
| 820 |
+
if [[ "$DYNAMIC_SOURCE_REFILL" == "1" ]]; then
|
| 821 |
+
DYNAMIC_SOURCE_REFILL_ARG=--train.dynamic-source-refill
|
| 822 |
+
else
|
| 823 |
+
DYNAMIC_SOURCE_REFILL_ARG=--train.no-dynamic-source-refill
|
| 824 |
+
fi
|
| 825 |
+
if [[ "$ACTION_RECOMPUTE" == "1" ]]; then
|
| 826 |
+
ACTION_RECOMPUTE_ARG=--train.full-source-gradient-checkpointing
|
| 827 |
+
else
|
| 828 |
+
ACTION_RECOMPUTE_ARG=--train.no-full-source-gradient-checkpointing
|
| 829 |
+
fi
|
| 830 |
+
if [[ "$RESUME_LOAD_FULL_STATE" != "0" && "$RESUME_LOAD_FULL_STATE" != "1" ]]; then
|
| 831 |
+
echo "[error] HFD_RESUME_LOAD_FULL_STATE must be 0 or 1, got $RESUME_LOAD_FULL_STATE" >&2
|
| 832 |
+
exit 2
|
| 833 |
+
fi
|
| 834 |
+
if [[ "$ALLOW_V11_WEIGHTS_INIT" != "0" && "$ALLOW_V11_WEIGHTS_INIT" != "1" ]]; then
|
| 835 |
+
echo "[error] HFD_ALLOW_V11_WEIGHTS_INIT must be 0 or 1, got $ALLOW_V11_WEIGHTS_INIT" >&2
|
| 836 |
+
exit 2
|
| 837 |
+
fi
|
| 838 |
+
if [[ "$ALLOW_V11_WEIGHTS_INIT" == "1" && "$RESUME_LOAD_FULL_STATE" == "1" ]]; then
|
| 839 |
+
echo "[error] v11 -> v12 initialization is weights-only; set HFD_RESUME_LOAD_FULL_STATE=0." >&2
|
| 840 |
+
exit 2
|
| 841 |
+
fi
|
| 842 |
+
if [[ "$WANDB_RESUME_MODE" != "never" \
|
| 843 |
+
&& "$WANDB_RESUME_MODE" != "allow" \
|
| 844 |
+
&& "$WANDB_RESUME_MODE" != "must" ]]; then
|
| 845 |
+
echo "[error] HFD_WANDB_RESUME must be never, allow or must." >&2
|
| 846 |
+
exit 2
|
| 847 |
+
fi
|
| 848 |
+
if [[ "$RESUME_LOAD_FULL_STATE" == "1" && "$RESUME_FROM_CHECKPOINT" == "None" ]]; then
|
| 849 |
+
echo "[error] Full-state resume requires HFD_RESUME_FROM_CHECKPOINT." >&2
|
| 850 |
+
exit 2
|
| 851 |
+
fi
|
| 852 |
+
if [[ "$RESUME_LOAD_FULL_STATE" == "1" ]]; then
|
| 853 |
+
RESUME_STATE_ARG=--train.resume-load-full-state
|
| 854 |
+
else
|
| 855 |
+
RESUME_STATE_ARG=--train.no-resume-load-full-state
|
| 856 |
+
fi
|
| 857 |
+
WANDB_ID_ARG=
|
| 858 |
+
if [[ -n "$WANDB_RUN_ID" ]]; then
|
| 859 |
+
WANDB_ID_ARG="--wandb.id=$WANDB_RUN_ID"
|
| 860 |
+
fi
|
| 861 |
+
|
| 862 |
+
python - \
|
| 863 |
+
"$FULL_DUPLEX_SEMANTICS_VERSION" \
|
| 864 |
+
"$JOINT_VELOCITY_LOSS_WEIGHT" \
|
| 865 |
+
"$JOINT_ACCELERATION_LOSS_WEIGHT" \
|
| 866 |
+
"$ROOT_CONTINUITY_LOSS_WEIGHT" \
|
| 867 |
+
"$STATIONARY_ROOT_HEIGHT_LOSS_WEIGHT" \
|
| 868 |
+
"$MOTION_DURATION_LOSS_WEIGHT" \
|
| 869 |
+
"$LATEST_INSTRUCTION_LOSS_WEIGHT" \
|
| 870 |
+
"$MIN_LR_RATE" \
|
| 871 |
+
"$INTERRUPTION_RATIO" \
|
| 872 |
+
"$INTERRUPTION_MIN_START_SECONDS" \
|
| 873 |
+
"$INTERRUPTION_MAX_START_SECONDS" \
|
| 874 |
+
"$INTERRUPTION_MIN_REMAINING_SECONDS" \
|
| 875 |
+
"$INTERRUPTION_TASK_SESSION_RATIO" \
|
| 876 |
+
"$CONTROL_TASK_SESSION_RATIO" \
|
| 877 |
+
"$BACKCHANNEL_TASK_SESSION_RATIO" \
|
| 878 |
+
"$HESITATION_TASK_WINDOW_RATIO" \
|
| 879 |
+
"$PAUSE_TASK_WINDOW_RATIO" \
|
| 880 |
+
"$HESITATION_FILLED_RATIO" \
|
| 881 |
+
"$CONTROL_FOLLOWUP_RATIO" \
|
| 882 |
+
"$CONTROL_POST_IDLE_MIN_STEPS" \
|
| 883 |
+
"$CONTROL_POST_IDLE_MAX_STEPS" \
|
| 884 |
+
"$STOP_ENTER_LOSS_WEIGHT" \
|
| 885 |
+
"$STOP_STAY_LOSS_WEIGHT" \
|
| 886 |
+
"$CONTROL_MIN_START_SECONDS" \
|
| 887 |
+
"$CONTROL_MAX_START_SECONDS" \
|
| 888 |
+
"$CONTROL_MIN_REMAINING_SECONDS" \
|
| 889 |
+
"$CONTROL_POST_IDLE_SAMPLING" \
|
| 890 |
+
"$STOP_RELEASE_LOSS_WEIGHT" \
|
| 891 |
+
"$SELF_REPAIR_IMPLICIT_RATIO" \
|
| 892 |
+
"$INSTRUCTION_RECENCY_SLOTS" \
|
| 893 |
+
"$INSTRUCTION_MEMORY_LOSS_WEIGHT" <<'PY'
|
| 894 |
+
import math
|
| 895 |
+
import sys
|
| 896 |
+
|
| 897 |
+
version = int(sys.argv[1])
|
| 898 |
+
smoothness_weights = tuple(float(value) for value in sys.argv[2:5])
|
| 899 |
+
stationary_weight = float(sys.argv[5])
|
| 900 |
+
duration_weight = float(sys.argv[6])
|
| 901 |
+
instruction_weight = float(sys.argv[7])
|
| 902 |
+
min_lr_rate = float(sys.argv[8])
|
| 903 |
+
interruption_ratio = float(sys.argv[9])
|
| 904 |
+
interruption_min_start = float(sys.argv[10])
|
| 905 |
+
interruption_max_start = float(sys.argv[11])
|
| 906 |
+
interruption_min_remaining = float(sys.argv[12])
|
| 907 |
+
interruption_task_ratio = float(sys.argv[13])
|
| 908 |
+
control_task_ratio = float(sys.argv[14])
|
| 909 |
+
backchannel_task_ratio = float(sys.argv[15])
|
| 910 |
+
hesitation_window_ratio = float(sys.argv[16])
|
| 911 |
+
pause_window_ratio = float(sys.argv[17])
|
| 912 |
+
hesitation_filled_ratio = float(sys.argv[18])
|
| 913 |
+
control_followup_ratio = float(sys.argv[19])
|
| 914 |
+
control_post_idle_min_steps = int(sys.argv[20])
|
| 915 |
+
control_post_idle_max_steps = int(sys.argv[21])
|
| 916 |
+
stop_enter_weight = float(sys.argv[22])
|
| 917 |
+
stop_stay_weight = float(sys.argv[23])
|
| 918 |
+
control_min_start = float(sys.argv[24])
|
| 919 |
+
control_max_start = float(sys.argv[25])
|
| 920 |
+
control_min_remaining = float(sys.argv[26])
|
| 921 |
+
control_post_idle_sampling = sys.argv[27]
|
| 922 |
+
stop_release_weight = float(sys.argv[28])
|
| 923 |
+
self_repair_implicit_ratio = float(sys.argv[29])
|
| 924 |
+
instruction_recency_slots = int(sys.argv[30])
|
| 925 |
+
instruction_memory_weight = float(sys.argv[31])
|
| 926 |
+
if any(not math.isfinite(value) or value < 0.0 for value in (*smoothness_weights, stationary_weight, duration_weight, instruction_weight, instruction_memory_weight, stop_enter_weight, stop_stay_weight, stop_release_weight)):
|
| 927 |
+
raise SystemExit("[error] Auxiliary loss weights must be finite and non-negative")
|
| 928 |
+
if not math.isfinite(min_lr_rate) or not 0.0 <= min_lr_rate <= 1.0:
|
| 929 |
+
raise SystemExit("[error] HFD_MIN_LR_RATE must be finite and within [0, 1]")
|
| 930 |
+
if version == 9 and (any(value != 0.0 for value in smoothness_weights) or stationary_weight != 0.0):
|
| 931 |
+
raise SystemExit("[error] Semantics 9 requires all auxiliary loss weights to be zero")
|
| 932 |
+
if version in {11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24} and not all(value > 0.0 for value in smoothness_weights):
|
| 933 |
+
raise SystemExit("[error] Semantics 11-24 requires all smoothness weights to be positive")
|
| 934 |
+
if version == 11 and stationary_weight != 0.0:
|
| 935 |
+
raise SystemExit("[error] Semantics 11 requires stationary root-height weight zero")
|
| 936 |
+
if version in {12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24} and stationary_weight <= 0.0:
|
| 937 |
+
raise SystemExit("[error] Semantics 12-24 requires positive stationary root-height weight")
|
| 938 |
+
if (version in {16, 17, 18, 19, 20, 21, 22, 23, 24}) != (duration_weight > 0.0):
|
| 939 |
+
raise SystemExit("[error] Motion-duration loss must be positive only for semantics 16-24")
|
| 940 |
+
if (version in {18, 19, 20, 21, 22, 23, 24}) != (instruction_weight > 0.0):
|
| 941 |
+
raise SystemExit("[error] Latest-instruction loss must be positive only for semantics 18-24")
|
| 942 |
+
if version == 24:
|
| 943 |
+
memory_enabled = instruction_recency_slots == 3 and instruction_memory_weight > 0.0
|
| 944 |
+
memory_disabled = instruction_recency_slots == 0 and instruction_memory_weight == 0.0
|
| 945 |
+
if not (memory_enabled or memory_disabled):
|
| 946 |
+
raise SystemExit("[error] Semantics 24 instruction memory must use slots=3 with positive loss, or slots=0 with zero loss")
|
| 947 |
+
elif instruction_recency_slots != 0 or instruction_memory_weight != 0.0:
|
| 948 |
+
raise SystemExit("[error] Instruction memory retrieval is enabled only for semantics 24")
|
| 949 |
+
if version in {20, 21, 22, 23, 24}:
|
| 950 |
+
if version == 20 and not math.isclose(interruption_ratio, 0.5, rel_tol=0.0, abs_tol=1e-12):
|
| 951 |
+
raise SystemExit("[error] Semantics 20 requires HFD_INTERRUPTION_RATIO=0.5")
|
| 952 |
+
if (
|
| 953 |
+
not all(
|
| 954 |
+
math.isfinite(value) and value > 0.0
|
| 955 |
+
for value in (
|
| 956 |
+
interruption_min_start,
|
| 957 |
+
interruption_max_start,
|
| 958 |
+
interruption_min_remaining,
|
| 959 |
+
)
|
| 960 |
+
)
|
| 961 |
+
or interruption_min_start > interruption_max_start
|
| 962 |
+
):
|
| 963 |
+
raise SystemExit("[error] Invalid interruption timing")
|
| 964 |
+
if version in {21, 22, 23, 24}:
|
| 965 |
+
if not math.isfinite(control_task_ratio) or not 0.0 < control_task_ratio < 0.5:
|
| 966 |
+
raise SystemExit("[error] HFD_CONTROL_TASK_SESSION_RATIO must be within (0, 0.5)")
|
| 967 |
+
if not math.isfinite(control_followup_ratio) or not 0.0 <= control_followup_ratio <= 1.0:
|
| 968 |
+
raise SystemExit("[error] HFD_CONTROL_FOLLOWUP_RATIO must be within [0, 1]")
|
| 969 |
+
effective_post_idle_max = (
|
| 970 |
+
control_post_idle_max_steps
|
| 971 |
+
if control_post_idle_max_steps > 0
|
| 972 |
+
else 7
|
| 973 |
+
)
|
| 974 |
+
if not 1 <= control_post_idle_min_steps <= effective_post_idle_max <= 50:
|
| 975 |
+
raise SystemExit(
|
| 976 |
+
"[error] Stop post-idle steps must satisfy 1 <= min <= max <= 50"
|
| 977 |
+
)
|
| 978 |
+
if (
|
| 979 |
+
not all(
|
| 980 |
+
math.isfinite(value) and value > 0.0
|
| 981 |
+
for value in (
|
| 982 |
+
control_min_start,
|
| 983 |
+
control_max_start,
|
| 984 |
+
control_min_remaining,
|
| 985 |
+
)
|
| 986 |
+
)
|
| 987 |
+
or control_min_start > control_max_start
|
| 988 |
+
):
|
| 989 |
+
raise SystemExit("[error] Invalid emergency-stop timing")
|
| 990 |
+
if version in {22, 23, 24}:
|
| 991 |
+
if not math.isfinite(interruption_task_ratio) or not 0.0 < interruption_task_ratio < 1.0:
|
| 992 |
+
raise SystemExit("[error] HFD_INTERRUPTION_TASK_SESSION_RATIO must be within (0, 1)")
|
| 993 |
+
if not math.isfinite(backchannel_task_ratio) or not 0.0 <= backchannel_task_ratio < 1.0:
|
| 994 |
+
raise SystemExit("[error] HFD_BACKCHANNEL_TASK_SESSION_RATIO must be within [0, 1)")
|
| 995 |
+
if interruption_task_ratio + control_task_ratio + backchannel_task_ratio >= 1.0:
|
| 996 |
+
raise SystemExit("[error] Interruption, stop and backchannel ratios leave no turn-taking data")
|
| 997 |
+
if version in {23, 24}:
|
| 998 |
+
if (
|
| 999 |
+
not all(
|
| 1000 |
+
math.isfinite(value) and 0.0 < value < 1.0
|
| 1001 |
+
for value in (hesitation_window_ratio, pause_window_ratio)
|
| 1002 |
+
)
|
| 1003 |
+
or hesitation_window_ratio + pause_window_ratio >= 1.0
|
| 1004 |
+
):
|
| 1005 |
+
raise SystemExit("[error] Invalid v23 hesitation/pause window ratios")
|
| 1006 |
+
if not math.isfinite(hesitation_filled_ratio) or not 0.0 < hesitation_filled_ratio < 1.0:
|
| 1007 |
+
raise SystemExit("[error] HFD_HESITATION_FILLED_RATIO must be within (0, 1)")
|
| 1008 |
+
if version == 23 and self_repair_implicit_ratio != 0.0:
|
| 1009 |
+
raise SystemExit("[error] Semantics 23 requires HFD_SELF_REPAIR_IMPLICIT_RATIO=0")
|
| 1010 |
+
if version == 24 and not (
|
| 1011 |
+
math.isfinite(self_repair_implicit_ratio)
|
| 1012 |
+
and 0.0 < self_repair_implicit_ratio < 1.0
|
| 1013 |
+
):
|
| 1014 |
+
raise SystemExit("[error] Semantics 24 requires HFD_SELF_REPAIR_IMPLICIT_RATIO within (0, 1)")
|
| 1015 |
+
elif hesitation_window_ratio != 0.0 or pause_window_ratio != 0.0:
|
| 1016 |
+
raise SystemExit("[error] Hesitation/pause window ratios are semantics-23/24 only")
|
| 1017 |
+
if control_post_idle_sampling not in {"legacy_uniform", "short_heavy_long_tail"}:
|
| 1018 |
+
raise SystemExit("[error] Unsupported HFD_CONTROL_POST_IDLE_SAMPLING")
|
| 1019 |
+
stop_weights = (stop_enter_weight, stop_stay_weight, stop_release_weight)
|
| 1020 |
+
if all(weight == 0.0 for weight in stop_weights):
|
| 1021 |
+
if control_post_idle_sampling != "legacy_uniform":
|
| 1022 |
+
raise SystemExit("[error] Disabled Stop loss requires legacy idle sampling")
|
| 1023 |
+
elif not all(weight > 0.0 for weight in stop_weights):
|
| 1024 |
+
raise SystemExit("[error] Balanced Stop objective requires all three positive weights")
|
| 1025 |
+
elif control_post_idle_sampling != "short_heavy_long_tail":
|
| 1026 |
+
raise SystemExit("[error] Balanced Stop objective requires short-heavy idle sampling")
|
| 1027 |
+
elif interruption_task_ratio != 0.0:
|
| 1028 |
+
raise SystemExit("[error] Final interruption-session balancing is semantics-22/23/24 only")
|
| 1029 |
+
elif control_post_idle_sampling != "legacy_uniform" or stop_release_weight != 0.0:
|
| 1030 |
+
raise SystemExit("[error] Balanced Stop release policy is semantics-22/23/24 only")
|
| 1031 |
+
elif interruption_task_ratio != 0.0 or control_task_ratio != 0.0 or backchannel_task_ratio != 0.0 or control_followup_ratio != 0.0 or control_post_idle_min_steps != 1 or control_post_idle_max_steps != 0 or control_post_idle_sampling != "legacy_uniform" or stop_enter_weight != 0.0 or stop_stay_weight != 0.0 or stop_release_weight != 0.0 or self_repair_implicit_ratio != 0.0:
|
| 1032 |
+
raise SystemExit("[error] Balanced control-task policy is available only in semantics 21-24")
|
| 1033 |
+
PY
|
| 1034 |
+
|
| 1035 |
+
if ! [[ "$SOURCE_LENGTH_LOOKAHEAD_BATCHES" =~ ^[1-9][0-9]*$ ]]; then
|
| 1036 |
+
echo "[error] HFD_SOURCE_LENGTH_LOOKAHEAD_BATCHES must be a positive integer" >&2
|
| 1037 |
+
exit 2
|
| 1038 |
+
fi
|
| 1039 |
+
if ! [[ "$VAL_BATCH_SIZE" =~ ^[1-9][0-9]*$ ]]; then
|
| 1040 |
+
echo "[error] HFD_VAL_BATCH_SIZE must be a positive integer" >&2
|
| 1041 |
+
exit 2
|
| 1042 |
+
fi
|
| 1043 |
+
|
| 1044 |
+
if [[ -z "${PSI_DATALOADER_PREFETCH_FACTOR:-}" ]]; then
|
| 1045 |
+
if (( NUM_WORKERS > 0 )); then
|
| 1046 |
+
PREFETCH_BATCHES=$SOURCE_LENGTH_LOOKAHEAD_BATCHES
|
| 1047 |
+
# Exact bucketing consumes a complete lookahead pool before yielding it to
|
| 1048 |
+
# the GPU. Keep one complete pool under construction; while the wrapper
|
| 1049 |
+
# drains it, workers refill the same bounded queue for the next group.
|
| 1050 |
+
PSI_DATALOADER_PREFETCH_FACTOR=$(( (PREFETCH_BATCHES * TRAIN_BATCH_SIZE + NUM_WORKERS - 1) / NUM_WORKERS ))
|
| 1051 |
+
else
|
| 1052 |
+
PSI_DATALOADER_PREFETCH_FACTOR=1
|
| 1053 |
+
fi
|
| 1054 |
+
fi
|
| 1055 |
+
export PSI_DATALOADER_PREFETCH_FACTOR
|
| 1056 |
+
|
| 1057 |
+
if [[ "${HFD_CONFIG_ONLY:-0}" == "1" ]]; then
|
| 1058 |
+
echo "[HFDv2 config] mode=$EXPERIMENT_MODE"
|
| 1059 |
+
echo "[HFDv2 config] limit_sources=$LIMIT_SOURCES"
|
| 1060 |
+
echo "[HFDv2 config] microbatch_per_gpu=$TRAIN_BATCH_SIZE"
|
| 1061 |
+
echo "[HFDv2 config] exact_source_length_lookahead_batches=$SOURCE_LENGTH_LOOKAHEAD_BATCHES"
|
| 1062 |
+
echo "[HFDv2 config] compact_inactive_action_rows=$COMPACT_INACTIVE_ACTION_ROWS"
|
| 1063 |
+
echo "[HFDv2 config] dynamic_source_refill=$DYNAMIC_SOURCE_REFILL"
|
| 1064 |
+
echo "[HFDv2 config] dynamic_source_context_reserve_tokens=$DYNAMIC_SOURCE_CONTEXT_RESERVE_TOKENS"
|
| 1065 |
+
echo "[HFDv2 config] control_head_lr_multiplier=$CONTROL_HEAD_LR_MULTIPLIER"
|
| 1066 |
+
echo "[HFDv2 config] source_accumulation_groups=$SOURCE_ACCUMULATION_GROUPS"
|
| 1067 |
+
echo "[HFDv2 config] effective_sources_per_gpu=$EFFECTIVE_SOURCES_PER_GPU"
|
| 1068 |
+
echo "[HFDv2 config] action_recompute=$ACTION_RECOMPUTE"
|
| 1069 |
+
echo "[HFDv2 config] rtc_max_delay=11"
|
| 1070 |
+
echo "[HFDv2 config] action_stride_frames=1 qwen_stride_frames=20"
|
| 1071 |
+
echo "[HFDv2 config] max_stream_steps=$MAX_STREAM_STEPS"
|
| 1072 |
+
echo "[HFDv2 config] semantics_version=$FULL_DUPLEX_SEMANTICS_VERSION"
|
| 1073 |
+
echo "[HFDv2 config] serial_control=qwen_audio_kv_summary -> token -> action state_aware=$([[ "$FULL_DUPLEX_SEMANTICS_VERSION" == "21" ]] && echo 1 || echo 0)"
|
| 1074 |
+
echo "[HFDv2 config] smoothness_weights=$JOINT_VELOCITY_LOSS_WEIGHT,$JOINT_ACCELERATION_LOSS_WEIGHT,$ROOT_CONTINUITY_LOSS_WEIGHT"
|
| 1075 |
+
echo "[HFDv2 config] stationary_root_height_weight=$STATIONARY_ROOT_HEIGHT_LOSS_WEIGHT"
|
| 1076 |
+
echo "[HFDv2 config] motion_duration_weight=$MOTION_DURATION_LOSS_WEIGHT distribution_bins=$MOTION_DURATION_DISTRIBUTION_BINS range_seconds=$MOTION_DURATION_MIN_SECONDS-$MOTION_DURATION_MAX_SECONDS"
|
| 1077 |
+
echo "[HFDv2 config] dense_window_accumulation_steps=$DENSE_WINDOW_ACCUMULATION_STEPS"
|
| 1078 |
+
echo "[HFDv2 config] latest_instruction_weight=$LATEST_INSTRUCTION_LOSS_WEIGHT vad=0 inference_text_decoder=0"
|
| 1079 |
+
echo "[HFDv2 config] instruction_memory_weight=$INSTRUCTION_MEMORY_LOSS_WEIGHT recency_slots=$INSTRUCTION_RECENCY_SLOTS targets=previous/first training_only=1"
|
| 1080 |
+
echo "[HFDv2 config] stationary_heading_alignment=$([[ "$FULL_DUPLEX_SEMANTICS_VERSION" == "15" || "$FULL_DUPLEX_SEMANTICS_VERSION" == "16" || "$FULL_DUPLEX_SEMANTICS_VERSION" == "17" || "$FULL_DUPLEX_SEMANTICS_VERSION" == "18" || "$FULL_DUPLEX_SEMANTICS_VERSION" == "19" || "$FULL_DUPLEX_SEMANTICS_VERSION" == "20" || "$FULL_DUPLEX_SEMANTICS_VERSION" == "21" || "$FULL_DUPLEX_SEMANTICS_VERSION" == "22" || "$FULL_DUPLEX_SEMANTICS_VERSION" == "23" || "$FULL_DUPLEX_SEMANTICS_VERSION" == "24" ]] && echo 1 || echo 0)"
|
| 1081 |
+
echo "[HFDv2 config] safe_stationary_anchors=$([[ "$FULL_DUPLEX_SEMANTICS_VERSION" == "22" || "$FULL_DUPLEX_SEMANTICS_VERSION" == "23" || "$FULL_DUPLEX_SEMANTICS_VERSION" == "24" ]] && echo 1 || echo 0) dynamic_task_entry_blend=$([[ "$FULL_DUPLEX_SEMANTICS_VERSION" == "22" || "$FULL_DUPLEX_SEMANTICS_VERSION" == "23" || "$FULL_DUPLEX_SEMANTICS_VERSION" == "24" ]] && echo 1 || echo 0) idle_residual_retargeting=$([[ "$FULL_DUPLEX_SEMANTICS_VERSION" == "24" ]] && echo 1 || echo 0)"
|
| 1082 |
+
echo "[HFDv2 config] long_session_drop_ratio=$LONG_SESSION_DROP_RATIO"
|
| 1083 |
+
echo "[HFDv2 config] session_turns=$SESSION_MIN_TURNS-$SESSION_MAX_TURNS"
|
| 1084 |
+
echo "[HFDv2 config] session_voice_variants=$SESSION_VOICE_VARIANTS"
|
| 1085 |
+
echo "[HFDv2 config] initial_idle_steps=0-$INITIAL_IDLE_MAX_STEPS"
|
| 1086 |
+
echo "[HFDv2 config] turn_transition_frames=$TURN_TRANSITION_FRAMES"
|
| 1087 |
+
echo "[HFDv2 config] causal_listen_timing=$([[ "$FULL_DUPLEX_SEMANTICS_VERSION" == "14" || "$FULL_DUPLEX_SEMANTICS_VERSION" == "15" || "$FULL_DUPLEX_SEMANTICS_VERSION" == "16" || "$FULL_DUPLEX_SEMANTICS_VERSION" == "17" || "$FULL_DUPLEX_SEMANTICS_VERSION" == "18" || "$FULL_DUPLEX_SEMANTICS_VERSION" == "19" || "$FULL_DUPLEX_SEMANTICS_VERSION" == "20" || "$FULL_DUPLEX_SEMANTICS_VERSION" == "21" || "$FULL_DUPLEX_SEMANTICS_VERSION" == "22" || "$FULL_DUPLEX_SEMANTICS_VERSION" == "23" || "$FULL_DUPLEX_SEMANTICS_VERSION" == "24" ]] && echo 1 || echo 0)"
|
| 1088 |
+
echo "[HFDv2 config] interruption=$([[ "$FULL_DUPLEX_SEMANTICS_VERSION" == "20" || "$FULL_DUPLEX_SEMANTICS_VERSION" == "21" || "$FULL_DUPLEX_SEMANTICS_VERSION" == "22" || "$FULL_DUPLEX_SEMANTICS_VERSION" == "23" || "$FULL_DUPLEX_SEMANTICS_VERSION" == "24" ]] && echo 1 || echo 0) ratio=$INTERRUPTION_RATIO start_seconds=$INTERRUPTION_MIN_START_SECONDS-$INTERRUPTION_MAX_START_SECONDS min_remaining_seconds=$INTERRUPTION_MIN_REMAINING_SECONDS"
|
| 1089 |
+
echo "[HFDv2 config] task_variants=turn-taking:$TURN_TAKING_VARIANTS,interruption:$COMMAND_INTERRUPTION_VARIANTS,emergency-stop:$EMERGENCY_STOP_VARIANTS,safe-hold:$SAFE_HOLD_VARIANTS,backchannel:$BACKCHANNEL_VARIANTS,hesitation:$HESITATION_VARIANTS,pause:$PAUSE_VARIANTS"
|
| 1090 |
+
echo "[HFDv2 config] final_task_ratio_targets=legacy_sessions(turn-taking:auto,interruption:$INTERRUPTION_TASK_SESSION_RATIO,emergency-stop:$CONTROL_TASK_SESSION_RATIO,safe-hold:0,backchannel:$BACKCHANNEL_TASK_SESSION_RATIO),new_windows(hesitation:$HESITATION_TASK_WINDOW_RATIO,pause:$PAUSE_TASK_WINDOW_RATIO)"
|
| 1091 |
+
if [[ "$FULL_DUPLEX_SEMANTICS_VERSION" == "23" || "$FULL_DUPLEX_SEMANTICS_VERSION" == "24" ]]; then
|
| 1092 |
+
echo "[HFDv2 config] disfluency=listen_until_final_intent same_session_voice=1 filled_ratio=$HESITATION_FILLED_RATIO implicit_repair_ratio=$SELF_REPAIR_IMPLICIT_RATIO row_caps=$NEW_TASK_MAX_ROWS_PER_INSTRUCTION_TRAIN/$NEW_TASK_MAX_ROWS_PER_INSTRUCTION_VAL manifest=$HESITATION_PAUSE_MANIFEST"
|
| 1093 |
+
else
|
| 1094 |
+
echo "[HFDv2 config] disfluency=disabled"
|
| 1095 |
+
fi
|
| 1096 |
+
echo "[HFDv2 config] backchannel=motion_unchanged same_session_voice=1 start_seconds=$BACKCHANNEL_MIN_START_SECONDS-$BACKCHANNEL_MAX_START_SECONDS min_remaining_seconds=$BACKCHANNEL_MIN_REMAINING_SECONDS"
|
| 1097 |
+
echo "[HFDv2 config] control_session_ratio_each=$CONTROL_TASK_SESSION_RATIO followup_ratio=$CONTROL_FOLLOWUP_RATIO post_idle_steps=$CONTROL_POST_IDLE_MIN_STEPS-${CONTROL_POST_IDLE_MAX_STEPS:-0} sampling=$CONTROL_POST_IDLE_SAMPLING"
|
| 1098 |
+
echo "[HFDv2 config] stop_loss_weights=enter:$STOP_ENTER_LOSS_WEIGHT,no_motion:$STOP_STAY_LOSS_WEIGHT,release:$STOP_RELEASE_LOSS_WEIGHT event_normalized=1"
|
| 1099 |
+
echo "[HFDv2 config] control_timing=start:$CONTROL_MIN_START_SECONDS-$CONTROL_MAX_START_SECONDS remaining_after_heard:$CONTROL_MIN_REMAINING_SECONDS activity_aware=$CONTROL_ACTIVITY_AWARE_TIMING"
|
| 1100 |
+
echo "[HFDv2 config] filter_known_tts_translation_mismatches=$FILTER_KNOWN_TTS_TRANSLATION_MISMATCHES"
|
| 1101 |
+
echo "[HFDv2 config] canonical_motion_manifest=${CANONICAL_MOTION_MANIFEST:-disabled}"
|
| 1102 |
+
echo "[HFDv2 config] max_speech_onset_steps=$MAX_SPEECH_ONSET_STEPS"
|
| 1103 |
+
echo "[HFDv2 config] listen_reaction_steps=$LISTEN_REACTION_STEPS"
|
| 1104 |
+
echo "[HFDv2 config] validation=global_sources:$VAL_NUM_BATCHES local_batch:$VAL_BATCH_SIZE every_steps:$VALIDATION_STEPS"
|
| 1105 |
+
echo "[HFDv2 config] best_checkpoint_min_relative_delta=$BEST_CHECKPOINT_MIN_RELATIVE_DELTA"
|
| 1106 |
+
echo "[HFDv2 config] wandb_verbose_metrics=$HFD_WANDB_VERBOSE_METRICS"
|
| 1107 |
+
echo "[HFDv2 config] max_training_steps=$MAX_TRAINING_STEPS"
|
| 1108 |
+
echo "[HFDv2 config] checkpointing_steps=$CHECKPOINTING_STEPS"
|
| 1109 |
+
echo "[HFDv2 config] cosine_min_lr_rate=$MIN_LR_RATE"
|
| 1110 |
+
echo "[HFDv2 config] warmup_steps=$WARMUP_STEPS"
|
| 1111 |
+
echo "[HFDv2 config] resume_load_full_state=$RESUME_LOAD_FULL_STATE"
|
| 1112 |
+
echo "[HFDv2 config] allow_v11_weights_init=$ALLOW_V11_WEIGHTS_INIT"
|
| 1113 |
+
echo "[HFDv2 config] wandb_resume=$WANDB_RESUME_MODE"
|
| 1114 |
+
echo "[HFDv2 config] wandb_group=$WANDB_GROUP"
|
| 1115 |
+
echo "[HFDv2 config] run_timestamp=$RUN_TIMESTAMP"
|
| 1116 |
+
exit 0
|
| 1117 |
+
fi
|
| 1118 |
+
|
| 1119 |
+
for path in \
|
| 1120 |
+
"$BACKGROUND_ASSIGNMENT_PATH" \
|
| 1121 |
+
"$BACKGROUND_MANIFEST" \
|
| 1122 |
+
"$PRODUCTION_MARKER" \
|
| 1123 |
+
"$MOTION_METADATA" \
|
| 1124 |
+
"$AUDIO_MANIFEST" \
|
| 1125 |
+
"$BACKBONE/config.json"; do
|
| 1126 |
+
if [[ ! -f "$path" ]]; then
|
| 1127 |
+
echo "[error] Required HFDv2 input is missing: $path" >&2
|
| 1128 |
+
exit 2
|
| 1129 |
+
fi
|
| 1130 |
+
done
|
| 1131 |
+
if [[ ! -d "$CONTINUOUS_DATA_ROOT/mixed_waveforms" || ! -d "$CONTINUOUS_DATA_ROOT/background" ]]; then
|
| 1132 |
+
echo "[error] HFDv2 raw mixed/background waveform roots are incomplete under $CONTINUOUS_DATA_ROOT" >&2
|
| 1133 |
+
exit 2
|
| 1134 |
+
fi
|
| 1135 |
+
|
| 1136 |
+
python - \
|
| 1137 |
+
"$CONTINUOUS_DATA_ROOT" \
|
| 1138 |
+
"$BACKGROUND_ASSIGNMENT_PATH" \
|
| 1139 |
+
"$BACKGROUND_MANIFEST" \
|
| 1140 |
+
"$PRODUCTION_MARKER" \
|
| 1141 |
+
"5" \
|
| 1142 |
+
"$STATIONARY_BLEND_FRAMES" <<'PY'
|
| 1143 |
+
import hashlib
|
| 1144 |
+
import json
|
| 1145 |
+
import sys
|
| 1146 |
+
import wave
|
| 1147 |
+
from pathlib import Path
|
| 1148 |
+
|
| 1149 |
+
(
|
| 1150 |
+
root_value,
|
| 1151 |
+
assignment_value,
|
| 1152 |
+
background_manifest_value,
|
| 1153 |
+
marker_value,
|
| 1154 |
+
expected_idle,
|
| 1155 |
+
expected_blend,
|
| 1156 |
+
) = sys.argv[1:]
|
| 1157 |
+
root = Path(root_value).resolve()
|
| 1158 |
+
assignment_path = Path(assignment_value).resolve()
|
| 1159 |
+
marker = json.loads(Path(marker_value).read_text(encoding="utf-8"))
|
| 1160 |
+
if marker.get("format") != "hfd_continuous_office_production_v1":
|
| 1161 |
+
raise RuntimeError(f"Unsupported production marker: {marker_value}")
|
| 1162 |
+
assignment_payload = assignment_path.read_bytes()
|
| 1163 |
+
if hashlib.sha256(assignment_payload).hexdigest() != marker.get("assignment_sha256"):
|
| 1164 |
+
raise RuntimeError("Continuous-background assignment changed after production validation")
|
| 1165 |
+
if int(marker.get("initial_idle_max_steps", -1)) != int(expected_idle):
|
| 1166 |
+
raise RuntimeError(
|
| 1167 |
+
"Raw source-corpus idle reservation differs from its production contract"
|
| 1168 |
+
)
|
| 1169 |
+
if int(marker.get("stationary_blend_frames", -1)) != int(expected_blend):
|
| 1170 |
+
raise RuntimeError("HFD_STATIONARY_BLEND_FRAMES differs from the production contract")
|
| 1171 |
+
|
| 1172 |
+
backgrounds = {}
|
| 1173 |
+
for line in Path(background_manifest_value).read_text(encoding="utf-8").splitlines():
|
| 1174 |
+
if not line.strip():
|
| 1175 |
+
continue
|
| 1176 |
+
row = json.loads(line)
|
| 1177 |
+
backgrounds[str(row["filename"])] = root / "background" / str(row["audio_path"])
|
| 1178 |
+
|
| 1179 |
+
source_count = 0
|
| 1180 |
+
sentinel_rows = []
|
| 1181 |
+
for line in assignment_payload.decode("utf-8").splitlines():
|
| 1182 |
+
if not line.strip():
|
| 1183 |
+
continue
|
| 1184 |
+
row = json.loads(line)
|
| 1185 |
+
if source_count == 0:
|
| 1186 |
+
sentinel_rows.append(row)
|
| 1187 |
+
elif source_count == int(marker.get("source_count", -1)) - 1:
|
| 1188 |
+
sentinel_rows.append(row)
|
| 1189 |
+
source_count += 1
|
| 1190 |
+
if source_count != int(marker.get("source_count", -1)):
|
| 1191 |
+
raise RuntimeError(
|
| 1192 |
+
f"Assignment source count changed: {source_count} != {marker.get('source_count')}"
|
| 1193 |
+
)
|
| 1194 |
+
for row in sentinel_rows:
|
| 1195 |
+
mixed = Path(str(row["mixed_audio_path"]))
|
| 1196 |
+
background = backgrounds.get(str(row["background_key"]))
|
| 1197 |
+
if not mixed.is_file() or background is None or not background.is_file():
|
| 1198 |
+
raise FileNotFoundError(
|
| 1199 |
+
f"Missing raw HFDv2 waveform sentinel: mixed={mixed} background={background}"
|
| 1200 |
+
)
|
| 1201 |
+
for wav_path in (mixed, background):
|
| 1202 |
+
with wave.open(str(wav_path), "rb") as handle:
|
| 1203 |
+
geometry = (
|
| 1204 |
+
handle.getnchannels(),
|
| 1205 |
+
handle.getsampwidth(),
|
| 1206 |
+
handle.getframerate(),
|
| 1207 |
+
)
|
| 1208 |
+
if geometry != (1, 2, 16_000):
|
| 1209 |
+
raise RuntimeError(f"Expected mono PCM16 16kHz WAV, got {geometry}: {wav_path}")
|
| 1210 |
+
print(
|
| 1211 |
+
f"[ready] HFDv2 raw waveform contract verified: sources={source_count}, "
|
| 1212 |
+
f"assignment_sha256={marker['assignment_sha256'][:12]}..."
|
| 1213 |
+
)
|
| 1214 |
+
PY
|
| 1215 |
+
export HFD_CONTINUOUS_DATA_CONTRACT_PATH="$PRODUCTION_MARKER"
|
| 1216 |
+
export HFD_CONTINUOUS_DATA_ROOT="$CONTINUOUS_DATA_ROOT"
|
| 1217 |
+
export HFD_BACKGROUND_ASSIGNMENT_PATH="$BACKGROUND_ASSIGNMENT_PATH"
|
| 1218 |
+
|
| 1219 |
+
DETECTED_GPU_COUNT=$(python - <<'PY'
|
| 1220 |
+
import torch
|
| 1221 |
+
print(torch.cuda.device_count())
|
| 1222 |
+
PY
|
| 1223 |
+
)
|
| 1224 |
+
if [[ ! "$DETECTED_GPU_COUNT" =~ ^[1-9][0-9]*$ ]]; then
|
| 1225 |
+
echo "[error] PyTorch did not detect a usable CUDA GPU (device_count=$DETECTED_GPU_COUNT)." >&2
|
| 1226 |
+
exit 2
|
| 1227 |
+
fi
|
| 1228 |
+
NPROC_PER_NODE=${HFD_NPROC_PER_NODE:-$DETECTED_GPU_COUNT}
|
| 1229 |
+
if [[ ! "$NPROC_PER_NODE" =~ ^[1-9][0-9]*$ ]] \
|
| 1230 |
+
|| (( NPROC_PER_NODE > DETECTED_GPU_COUNT )); then
|
| 1231 |
+
echo "[error] HFD_NPROC_PER_NODE=$NPROC_PER_NODE exceeds detected GPUs=$DETECTED_GPU_COUNT." >&2
|
| 1232 |
+
exit 2
|
| 1233 |
+
fi
|
| 1234 |
+
if [[ "$EXPERIMENT_MODE" == "overfit50" ]] \
|
| 1235 |
+
&& (( TRAIN_BATCH_SIZE * NPROC_PER_NODE > LIMIT_SOURCES )); then
|
| 1236 |
+
echo "[error] HFDv2 overfit50 cannot form one rank-aligned microgroup: " >&2
|
| 1237 |
+
echo " batch=$TRAIN_BATCH_SIZE GPUs=$NPROC_PER_NODE sources=$LIMIT_SOURCES" >&2
|
| 1238 |
+
exit 2
|
| 1239 |
+
fi
|
| 1240 |
+
TORCHRUN_LOCAL_ADDR=${HFD_TORCHRUN_LOCAL_ADDR:-127.0.0.1}
|
| 1241 |
+
|
| 1242 |
+
if [[ "${REPORT_TO,,}" == "wandb" ]]; then
|
| 1243 |
+
if [[ -z "${WANDB_API_KEY:-}" ]]; then
|
| 1244 |
+
echo "[error] WandB is enabled but WANDB_API_KEY is empty." >&2
|
| 1245 |
+
exit 2
|
| 1246 |
+
fi
|
| 1247 |
+
python - <<'PY'
|
| 1248 |
+
import os
|
| 1249 |
+
import sys
|
| 1250 |
+
import wandb
|
| 1251 |
+
try:
|
| 1252 |
+
if wandb.Api(timeout=int(os.environ.get("HFD_WANDB_AUTH_TIMEOUT", "30"))).viewer is None:
|
| 1253 |
+
raise RuntimeError("WandB returned no authenticated user")
|
| 1254 |
+
except Exception as exc:
|
| 1255 |
+
print(f"[error] WandB authentication failed ({type(exc).__name__}): {exc}", file=sys.stderr)
|
| 1256 |
+
sys.exit(2)
|
| 1257 |
+
print("[ready] WandB authentication verified.")
|
| 1258 |
+
PY
|
| 1259 |
+
fi
|
| 1260 |
+
|
| 1261 |
+
GPU_MONITOR_ROOT=${HFD_GPU_MONITOR_ROOT:-/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/monitor}
|
| 1262 |
+
MONITOR_CMD=$GPU_MONITOR_ROOT/cmd.sh
|
| 1263 |
+
MONITOR_SCRIPT=$GPU_MONITOR_ROOT/monitor.sh
|
| 1264 |
+
MONITOR_PY=$GPU_MONITOR_ROOT/monitor.py
|
| 1265 |
+
MONITOR_GUARD_PID=
|
| 1266 |
+
MONITOR_GUARD_SECONDS=${HFD_MONITOR_GUARD_SECONDS:-5}
|
| 1267 |
+
stop_gpu_monitor() {
|
| 1268 |
+
pkill -f "$MONITOR_SCRIPT" 2>/dev/null || true
|
| 1269 |
+
pkill -f "$MONITOR_PY" 2>/dev/null || true
|
| 1270 |
+
}
|
| 1271 |
+
guard_gpu_monitor() {
|
| 1272 |
+
while true; do
|
| 1273 |
+
stop_gpu_monitor
|
| 1274 |
+
sleep "$MONITOR_GUARD_SECONDS"
|
| 1275 |
+
done
|
| 1276 |
+
}
|
| 1277 |
+
restore_gpu_monitor() {
|
| 1278 |
+
if [[ -n "${MONITOR_GUARD_PID:-}" ]]; then
|
| 1279 |
+
kill "$MONITOR_GUARD_PID" 2>/dev/null || true
|
| 1280 |
+
wait "$MONITOR_GUARD_PID" 2>/dev/null || true
|
| 1281 |
+
fi
|
| 1282 |
+
if [[ -f "$MONITOR_CMD" ]]; then
|
| 1283 |
+
set +e
|
| 1284 |
+
stop_gpu_monitor
|
| 1285 |
+
source "$MONITOR_CMD"
|
| 1286 |
+
set -e
|
| 1287 |
+
fi
|
| 1288 |
+
}
|
| 1289 |
+
trap restore_gpu_monitor EXIT
|
| 1290 |
+
stop_gpu_monitor
|
| 1291 |
+
guard_gpu_monitor &
|
| 1292 |
+
MONITOR_GUARD_PID=$!
|
| 1293 |
+
|
| 1294 |
+
echo "[HFD] input: raw 16kHz mixed-speech/background WAV, encoded online by Qwen2.5-Omni-3B"
|
| 1295 |
+
echo "[HFDv2] experiment mode: $EXPERIMENT_MODE"
|
| 1296 |
+
echo "[HFD] memory: every 0.4s window appends Qwen native KV; reset only at source boundary"
|
| 1297 |
+
echo "[HFD] action: 30-frame window / 20 new frames / 10-frame RTC prefix at 50fps"
|
| 1298 |
+
echo "[HFDv2] trainable: action/control + final $QWEN_TRAINABLE_LAST_LAYERS Qwen decoder layers + final norm"
|
| 1299 |
+
echo "[HFD] optimizer: average $DENSE_WINDOW_ACCUMULATION_STEPS consecutive 0.02s dense starts per update; Qwen condition reused for 20 starts"
|
| 1300 |
+
echo "[HFD] window batch: $TRAIN_BATCH_SIZE active sources/GPU, up to $((EFFECTIVE_SOURCES_PER_GPU * NPROC_PER_NODE)) global"
|
| 1301 |
+
echo "[HFD] exact source buckets: materialized lookahead=${SOURCE_LENGTH_LOOKAHEAD_BATCHES} local batches"
|
| 1302 |
+
echo "[HFD] eval: ${VAL_NUM_BATCHES} global sources, vectorized local batch=${VAL_BATCH_SIZE}"
|
| 1303 |
+
echo "[HFD] context guard: at most $MAX_STREAM_STEPS windows/source"
|
| 1304 |
+
echo "[HFD] session: $SESSION_MIN_TURNS-$SESSION_MAX_TURNS turns, one continuous background/KV lifetime"
|
| 1305 |
+
echo "[HFD] session voice: one identity/session; $SESSION_VOICE_VARIANTS static train variants"
|
| 1306 |
+
echo "[HFD] task variants: turn-taking=$TURN_TAKING_VARIANTS interruption=$COMMAND_INTERRUPTION_VARIANTS emergency-stop=$EMERGENCY_STOP_VARIANTS safe-hold=$SAFE_HOLD_VARIANTS backchannel=$BACKCHANNEL_VARIANTS hesitation=$HESITATION_VARIANTS pause=$PAUSE_VARIANTS; interruption=${INTERRUPTION_MIN_START_SECONDS}-${INTERRUPTION_MAX_START_SECONDS}s, leave>=${INTERRUPTION_MIN_REMAINING_SECONDS}s"
|
| 1307 |
+
echo "[HFD] backchannel: same-session voice over active motion, control/progress/RTC unchanged; onset=${BACKCHANNEL_MIN_START_SECONDS}-${BACKCHANNEL_MAX_START_SECONDS}s, leave>=${BACKCHANNEL_MIN_REMAINING_SECONDS}s"
|
| 1308 |
+
echo "[HFD] emergency stop: onset=${CONTROL_MIN_START_SECONDS}-${CONTROL_MAX_START_SECONDS}s, leave>=${CONTROL_MIN_REMAINING_SECONDS}s after command, activity_aware=$CONTROL_ACTIVITY_AWARE_TIMING"
|
| 1309 |
+
echo "[HFD] stop balance: post_idle_sampling=$CONTROL_POST_IDLE_SAMPLING enter=$STOP_ENTER_LOSS_WEIGHT no_motion=$STOP_STAY_LOSS_WEIGHT release=$STOP_RELEASE_LOSS_WEIGHT"
|
| 1310 |
+
echo "[HFD] semantic data audit: filter_known_tts_translation_mismatches=$FILTER_KNOWN_TTS_TRANSLATION_MISMATCHES"
|
| 1311 |
+
echo "[HFD] canonical motion per instruction: ${CANONICAL_MOTION_MANIFEST:-disabled}"
|
| 1312 |
+
echo "[HFD] final task targets: established-session mixture turn-taking=auto interruption=${INTERRUPTION_TASK_SESSION_RATIO} emergency-stop=${CONTROL_TASK_SESSION_RATIO} safe-hold=0 backchannel=${BACKCHANNEL_TASK_SESSION_RATIO}; appended window shares hesitation=${HESITATION_TASK_WINDOW_RATIO} pause=${PAUSE_TASK_WINDOW_RATIO}; followup=${CONTROL_FOLLOWUP_RATIO}"
|
| 1313 |
+
echo "[HFD] hesitation/pause: regenerated segments, one event/session, same voice/background/KV; filled_ratio=$HESITATION_FILLED_RATIO row_caps=$NEW_TASK_MAX_ROWS_PER_INSTRUCTION_TRAIN/$NEW_TASK_MAX_ROWS_PER_INSTRUCTION_VAL"
|
| 1314 |
+
echo "[HFD] session idle: initial 0-$INITIAL_IDLE_MAX_STEPS windows; task joins settle over $TURN_TRANSITION_FRAMES frames"
|
| 1315 |
+
echo "[HFD] source balance: fixed voice/session; filter voice windows at absolute=$MAX_VOICE_WINDOWS and median_ratio=$MAX_VOICE_MEDIAN_RATIO"
|
| 1316 |
+
echo "[HFDv2] action recompute: $ACTION_RECOMPUTE"
|
| 1317 |
+
echo "[HFD] serial control: train=$CONTROL_CONDITION_TRAIN_MODE inference=$CONTROL_CONDITION_INFERENCE_MODE temperature=$CONTROL_CONDITION_TEMPERATURE"
|
| 1318 |
+
echo "[HFD] task root: rigidly align every motion trajectory to the stationary anchor heading"
|
| 1319 |
+
echo "[HFD] smoothness: semantics=$FULL_DUPLEX_SEMANTICS_VERSION joint_velocity=$JOINT_VELOCITY_LOSS_WEIGHT joint_acceleration=$JOINT_ACCELERATION_LOSS_WEIGHT root_continuity=$ROOT_CONTINUITY_LOSS_WEIGHT"
|
| 1320 |
+
echo "[HFD] frame control: one label/action frame; v19+ supervises cross-phase tails and preserves the RTC prefix"
|
| 1321 |
+
echo "[HFD] stationary root-height weight: $STATIONARY_ROOT_HEIGHT_LOSS_WEIGHT"
|
| 1322 |
+
echo "[HFD] duration: predict once/listen->motion, GT-only train phase, weight=$MOTION_DURATION_LOSS_WEIGHT"
|
| 1323 |
+
echo "[HFD] latest instruction: exact TTS text CE once/listen->motion, weight=$LATEST_INSTRUCTION_LOSS_WEIGHT, VAD=off, inference decoder=off"
|
| 1324 |
+
echo "[HFD] instruction memory: latest/previous/first learned query markers; one previous/first text target per noninitial turn, weight=$INSTRUCTION_MEMORY_LOSS_WEIGHT, production KV read-only"
|
| 1325 |
+
echo "[HFD] gradient clip: $MAX_GRAD_NORM"
|
| 1326 |
+
echo "[HFD] LR: action_peak=$LEARNING_RATE qwen_peak=$QWEN_LR cosine_floor_rate=$MIN_LR_RATE"
|
| 1327 |
+
echo "[HFD] control head LR: ${CONTROL_HEAD_LR_MULTIPLIER}x action peak"
|
| 1328 |
+
echo "[HFD] LR warmup: $WARMUP_STEPS optimizer steps"
|
| 1329 |
+
echo "[HFD] source tail action compute: $([[ "$COMPACT_INACTIVE_ACTION_ROWS" == "1" ]] && echo active_rows_only || echo dense_padding_rows)"
|
| 1330 |
+
echo "[HFD] source slots: $([[ "$DYNAMIC_SOURCE_REFILL" == "1" ]] && echo refill_on_qwen_boundary || echo fixed_until_longest)"
|
| 1331 |
+
echo "[HFDv2] compact checkpoint: action/control + trainable Qwen delta; frozen 3B base is not duplicated"
|
| 1332 |
+
echo "[HFDv2] raw data: $CONTINUOUS_DATA_ROOT"
|
| 1333 |
+
echo "[HFDv2] GPU: $CUDA_VISIBLE_DEVICES; DDP processes: $NPROC_PER_NODE"
|
| 1334 |
+
echo "[HFDv2] workers: $NUM_WORKERS/rank, prefetch=${PSI_DATALOADER_PREFETCH_FACTOR}"
|
| 1335 |
+
echo "[HFDv2] WandB: project=$WANDB_PROJECT group=$WANDB_GROUP"
|
| 1336 |
+
echo "[HFDv2] timestamp=$RUN_TIMESTAMP resume=$RESUME_FROM_CHECKPOINT"
|
| 1337 |
+
echo "[HFDv2] full-state resume=$RESUME_LOAD_FULL_STATE WandB resume=$WANDB_RESUME_MODE"
|
| 1338 |
+
|
| 1339 |
+
args="posttrain_full_duplex_audio_command_psi0_config \
|
| 1340 |
+
--seed=42 \
|
| 1341 |
+
--exp=posttrain \
|
| 1342 |
+
--timestamp=$RUN_TIMESTAMP \
|
| 1343 |
+
--train.name=posttrain_full_duplex \
|
| 1344 |
+
--train.data_parallel=ddp \
|
| 1345 |
+
--train.mixed_precision=bf16 \
|
| 1346 |
+
--train.train_batch_size=$TRAIN_BATCH_SIZE \
|
| 1347 |
+
--train.val_batch_size=$VAL_BATCH_SIZE \
|
| 1348 |
+
--train.source-length-lookahead-batches=$SOURCE_LENGTH_LOOKAHEAD_BATCHES \
|
| 1349 |
+
$COMPACT_INACTIVE_ACTION_ROWS_ARG \
|
| 1350 |
+
$DYNAMIC_SOURCE_REFILL_ARG \
|
| 1351 |
+
--train.dynamic-source-context-reserve-tokens=$DYNAMIC_SOURCE_CONTEXT_RESERVE_TOKENS \
|
| 1352 |
+
--train.num-workers=$NUM_WORKERS \
|
| 1353 |
+
--train.val-num-workers=$VAL_NUM_WORKERS \
|
| 1354 |
+
--train.gradient_accumulation_steps=$GRADIENT_ACCUMULATION_STEPS \
|
| 1355 |
+
--train.resume_from_checkpoint=$RESUME_FROM_CHECKPOINT \
|
| 1356 |
+
$RESUME_STATE_ARG \
|
| 1357 |
+
--train.learning_rate=$LEARNING_RATE \
|
| 1358 |
+
--train.warmup-steps=$WARMUP_STEPS \
|
| 1359 |
+
--train.warmup-ratio=None \
|
| 1360 |
+
--train.max_training_steps=$MAX_TRAINING_STEPS \
|
| 1361 |
+
--train.checkpointing_steps=$CHECKPOINTING_STEPS \
|
| 1362 |
+
--train.validation_steps=$VALIDATION_STEPS \
|
| 1363 |
+
--train.val_num_batches=$VAL_NUM_BATCHES \
|
| 1364 |
+
--train.best-checkpoint-min-relative-delta=$BEST_CHECKPOINT_MIN_RELATIVE_DELTA \
|
| 1365 |
+
--train.max_checkpoints_to_keep=2 \
|
| 1366 |
+
--train.lr_scheduler_type=cosine_with_min_lr \
|
| 1367 |
+
--train.cosine-min-lr-rate=$MIN_LR_RATE \
|
| 1368 |
+
--train.lr_scheduler_kwargs.betas 0.9 0.999 \
|
| 1369 |
+
--train.lr_scheduler_kwargs.weight_decay=0 \
|
| 1370 |
+
--train.lr_scheduler_kwargs.eps=1e-8 \
|
| 1371 |
+
--train.full-duplex-semantics-version=$FULL_DUPLEX_SEMANTICS_VERSION \
|
| 1372 |
+
--train.tbptt-unroll-steps=$TBPTT_UNROLL_STEPS \
|
| 1373 |
+
--train.qwen-kv-gradient-horizon-windows=$QWEN_KV_GRADIENT_HORIZON \
|
| 1374 |
+
--train.dense-window-accumulation-steps=$DENSE_WINDOW_ACCUMULATION_STEPS \
|
| 1375 |
+
--train.source-accumulation-groups=$SOURCE_ACCUMULATION_GROUPS \
|
| 1376 |
+
--train.target-effective-sources-per-gpu=$EFFECTIVE_SOURCES_PER_GPU \
|
| 1377 |
+
$ACTION_RECOMPUTE_ARG \
|
| 1378 |
+
--train.control-loss-weight=$CONTROL_LOSS_WEIGHT \
|
| 1379 |
+
--train.control-head-lr-multiplier=$CONTROL_HEAD_LR_MULTIPLIER \
|
| 1380 |
+
--train.stop-enter-loss-weight=$STOP_ENTER_LOSS_WEIGHT \
|
| 1381 |
+
--train.stop-stay-loss-weight=$STOP_STAY_LOSS_WEIGHT \
|
| 1382 |
+
--train.stop-release-loss-weight=$STOP_RELEASE_LOSS_WEIGHT \
|
| 1383 |
+
--train.joint-velocity-loss-weight=$JOINT_VELOCITY_LOSS_WEIGHT \
|
| 1384 |
+
--train.joint-acceleration-loss-weight=$JOINT_ACCELERATION_LOSS_WEIGHT \
|
| 1385 |
+
--train.root-continuity-loss-weight=$ROOT_CONTINUITY_LOSS_WEIGHT \
|
| 1386 |
+
--train.stationary-root-height-loss-weight=$STATIONARY_ROOT_HEIGHT_LOSS_WEIGHT \
|
| 1387 |
+
--train.motion-duration-loss-weight=$MOTION_DURATION_LOSS_WEIGHT \
|
| 1388 |
+
--train.latest-instruction-loss-weight=$LATEST_INSTRUCTION_LOSS_WEIGHT \
|
| 1389 |
+
--train.instruction-memory-loss-weight=$INSTRUCTION_MEMORY_LOSS_WEIGHT \
|
| 1390 |
+
--train.max-grad-norm=$MAX_GRAD_NORM \
|
| 1391 |
+
--train.output_dir=$OUTPUT_DIR \
|
| 1392 |
+
--log.report_to=$REPORT_TO \
|
| 1393 |
+
--wandb.project=$WANDB_PROJECT \
|
| 1394 |
+
--wandb.group=$WANDB_GROUP \
|
| 1395 |
+
--wandb.resume=$WANDB_RESUME_MODE \
|
| 1396 |
+
$WANDB_ID_ARG \
|
| 1397 |
+
--data.online-audio \
|
| 1398 |
+
--data.continuous-audio-root=$CONTINUOUS_DATA_ROOT \
|
| 1399 |
+
--data.metadata-jsonl=$AUDIO_MANIFEST \
|
| 1400 |
+
$CANONICAL_MOTION_DATA_ARG \
|
| 1401 |
+
--data.motion-metadata-csv=$MOTION_METADATA \
|
| 1402 |
+
--data.motion-root=$MOTION_ROOT \
|
| 1403 |
+
--data.background-assignment-path=$BACKGROUND_ASSIGNMENT_PATH \
|
| 1404 |
+
--data.background-source-name=hfd_continuous_office_background \
|
| 1405 |
+
--data.background-task-name=audio_command_text \
|
| 1406 |
+
--data.action-exec-horizon=20 \
|
| 1407 |
+
--data.motion-fps=50 \
|
| 1408 |
+
--data.motion-metadata-fps=$MOTION_METADATA_FPS \
|
| 1409 |
+
--data.audio-sampling-rate=16000 \
|
| 1410 |
+
--data.audio-window-seconds=0.6 \
|
| 1411 |
+
--data.audio-sample-step-frames=20 \
|
| 1412 |
+
--data.action-sample-step-frames=1 \
|
| 1413 |
+
--data.motion-cache-size=$MOTION_CACHE_SIZE \
|
| 1414 |
+
--data.fixed-voice-per-record \
|
| 1415 |
+
--data.max-voice-windows=$MAX_VOICE_WINDOWS \
|
| 1416 |
+
--data.max-voice-median-ratio=$MAX_VOICE_MEDIAN_RATIO \
|
| 1417 |
+
--data.limit-sources=$LIMIT_SOURCES \
|
| 1418 |
+
--data.max-motion-steps=$MAX_MOTION_STEPS \
|
| 1419 |
+
--data.initial-idle-max-steps=$INITIAL_IDLE_MAX_STEPS \
|
| 1420 |
+
--data.stationary-blend-frames=$STATIONARY_BLEND_FRAMES \
|
| 1421 |
+
$STATIONARY_HEADING_DATA_ARG \
|
| 1422 |
+
$SAFE_STATIONARY_DATA_ARG \
|
| 1423 |
+
$STATIONARY_IDLE_RESIDUAL_DATA_ARG \
|
| 1424 |
+
$DYNAMIC_TASK_ENTRY_DATA_ARG \
|
| 1425 |
+
$MULTI_TURN_DATA_ARG \
|
| 1426 |
+
$CAUSAL_LISTEN_DATA_ARG \
|
| 1427 |
+
$INTERRUPTION_DATA_ARG \
|
| 1428 |
+
$INDEPENDENT_TASK_POOLS_ARG \
|
| 1429 |
+
--data.session-min-turns=$SESSION_MIN_TURNS \
|
| 1430 |
+
--data.session-max-turns=$SESSION_MAX_TURNS \
|
| 1431 |
+
--data.session-voice-variants=$SESSION_VOICE_VARIANTS \
|
| 1432 |
+
--data.turn-taking-session-variants=$TURN_TAKING_VARIANTS \
|
| 1433 |
+
--data.command-interruption-session-variants=$COMMAND_INTERRUPTION_VARIANTS \
|
| 1434 |
+
--data.emergency-stop-session-variants=$EMERGENCY_STOP_VARIANTS \
|
| 1435 |
+
--data.safe-hold-session-variants=$SAFE_HOLD_VARIANTS \
|
| 1436 |
+
--data.backchannel-session-variants=$BACKCHANNEL_VARIANTS \
|
| 1437 |
+
--data.hesitation-session-variants=$HESITATION_VARIANTS \
|
| 1438 |
+
--data.pause-session-variants=$PAUSE_VARIANTS \
|
| 1439 |
+
--data.interruption-task-session-ratio=$INTERRUPTION_TASK_SESSION_RATIO \
|
| 1440 |
+
--data.control-task-session-ratio=$CONTROL_TASK_SESSION_RATIO \
|
| 1441 |
+
--data.backchannel-task-session-ratio=$BACKCHANNEL_TASK_SESSION_RATIO \
|
| 1442 |
+
--data.hesitation-task-window-ratio=$HESITATION_TASK_WINDOW_RATIO \
|
| 1443 |
+
--data.pause-task-window-ratio=$PAUSE_TASK_WINDOW_RATIO \
|
| 1444 |
+
--data.control-followup-ratio=$CONTROL_FOLLOWUP_RATIO \
|
| 1445 |
+
--data.control-post-idle-min-steps=$CONTROL_POST_IDLE_MIN_STEPS \
|
| 1446 |
+
--data.control-post-idle-max-steps=$CONTROL_POST_IDLE_MAX_STEPS \
|
| 1447 |
+
--data.control-post-idle-sampling=$CONTROL_POST_IDLE_SAMPLING \
|
| 1448 |
+
--data.control-min-start-seconds=$CONTROL_MIN_START_SECONDS \
|
| 1449 |
+
--data.control-max-start-seconds=$CONTROL_MAX_START_SECONDS \
|
| 1450 |
+
--data.control-min-remaining-seconds=$CONTROL_MIN_REMAINING_SECONDS \
|
| 1451 |
+
$CONTROL_ACTIVITY_TIMING_ARG \
|
| 1452 |
+
$TRANSLATION_FILTER_DATA_ARG \
|
| 1453 |
+
--data.control-command-manifest=$CONTROL_COMMAND_MANIFEST \
|
| 1454 |
+
--data.backchannel-command-manifest=$BACKCHANNEL_COMMAND_MANIFEST \
|
| 1455 |
+
--data.hesitation-pause-manifest=$HESITATION_PAUSE_MANIFEST \
|
| 1456 |
+
--data.new-task-max-rows-per-instruction-train=$NEW_TASK_MAX_ROWS_PER_INSTRUCTION_TRAIN \
|
| 1457 |
+
--data.new-task-max-rows-per-instruction-val=$NEW_TASK_MAX_ROWS_PER_INSTRUCTION_VAL \
|
| 1458 |
+
--data.hesitation-filled-ratio=$HESITATION_FILLED_RATIO \
|
| 1459 |
+
--data.self-repair-implicit-ratio=$SELF_REPAIR_IMPLICIT_RATIO \
|
| 1460 |
+
--data.backchannel-min-start-seconds=$BACKCHANNEL_MIN_START_SECONDS \
|
| 1461 |
+
--data.backchannel-max-start-seconds=$BACKCHANNEL_MAX_START_SECONDS \
|
| 1462 |
+
--data.backchannel-min-remaining-seconds=$BACKCHANNEL_MIN_REMAINING_SECONDS \
|
| 1463 |
+
--data.safe-hold-stability-window-frames=$SAFE_HOLD_STABILITY_WINDOW_FRAMES \
|
| 1464 |
+
--data.interruption-ratio=$INTERRUPTION_RATIO \
|
| 1465 |
+
--data.interruption-min-start-seconds=$INTERRUPTION_MIN_START_SECONDS \
|
| 1466 |
+
--data.interruption-max-start-seconds=$INTERRUPTION_MAX_START_SECONDS \
|
| 1467 |
+
--data.interruption-min-remaining-seconds=$INTERRUPTION_MIN_REMAINING_SECONDS \
|
| 1468 |
+
--data.turn-transition-frames=$TURN_TRANSITION_FRAMES \
|
| 1469 |
+
--data.max-speech-onset-steps=$MAX_SPEECH_ONSET_STEPS \
|
| 1470 |
+
--data.listen-reaction-steps=$LISTEN_REACTION_STEPS \
|
| 1471 |
+
--data.max-stream-steps=$MAX_STREAM_STEPS \
|
| 1472 |
+
--data.long-session-drop-ratio=$LONG_SESSION_DROP_RATIO \
|
| 1473 |
+
--data.transform.repack.action-chunk-size=30 \
|
| 1474 |
+
--data.transform.repack.rtc-prefix-frames=10 \
|
| 1475 |
+
--data.transform.repack.textop-actions \
|
| 1476 |
+
--data.transform.repack.pad-action-dim=39 \
|
| 1477 |
+
--data.transform.repack.pad-state-dim=39 \
|
| 1478 |
+
--data.transform.field.action_norm_type=bounds_q99 \
|
| 1479 |
+
--data.transform.field.stat-path=assets/stats/seed_all_g1_motion39_textop_stats.json \
|
| 1480 |
+
--data.transform.field.normalize-state \
|
| 1481 |
+
--data.transform.field.pad-action-dim=39 \
|
| 1482 |
+
--data.transform.field.pad-state-dim=39 \
|
| 1483 |
+
--data.transform.model.text-only \
|
| 1484 |
+
--data.transform.model.audio-only \
|
| 1485 |
+
--data.transform.model.no-img-aug \
|
| 1486 |
+
--model.model_name_or_path=$BACKBONE \
|
| 1487 |
+
--model.tune-vlm \
|
| 1488 |
+
--model.no-gradient-checkpointing \
|
| 1489 |
+
--model.lang-backbone-lr=$QWEN_LR \
|
| 1490 |
+
--model.streaming-qwen-enabled \
|
| 1491 |
+
--model.streaming-qwen-trainable-last-layers=$QWEN_TRAINABLE_LAST_LAYERS \
|
| 1492 |
+
--model.streaming-qwen-train-final-norm \
|
| 1493 |
+
--model.instruction-recency-slots=$INSTRUCTION_RECENCY_SLOTS \
|
| 1494 |
+
--model.control-condition-train-mode=$CONTROL_CONDITION_TRAIN_MODE \
|
| 1495 |
+
--model.control-condition-inference-mode=$CONTROL_CONDITION_INFERENCE_MODE \
|
| 1496 |
+
--model.control-condition-temperature=$CONTROL_CONDITION_TEMPERATURE \
|
| 1497 |
+
$STATE_AWARE_CONTROL_MODEL_ARG \
|
| 1498 |
+
$MOTION_DURATION_MODEL_ARG \
|
| 1499 |
+
--model.motion-duration-default-seconds=$MOTION_DURATION_DEFAULT_SECONDS \
|
| 1500 |
+
--model.motion-duration-distribution-bins=$MOTION_DURATION_DISTRIBUTION_BINS \
|
| 1501 |
+
--model.motion-duration-min-seconds=$MOTION_DURATION_MIN_SECONDS \
|
| 1502 |
+
--model.motion-duration-max-seconds=$MOTION_DURATION_MAX_SECONDS \
|
| 1503 |
+
--model.noise-scheduler=flow \
|
| 1504 |
+
--model.action-chunk-size=30 \
|
| 1505 |
+
--model.action-exec-horizon=20 \
|
| 1506 |
+
--model.action-dim=39 \
|
| 1507 |
+
--model.observation-horizon=1 \
|
| 1508 |
+
--model.odim=39 \
|
| 1509 |
+
--model.view_feature_dim=2048 \
|
| 1510 |
+
--model.use-future-progress-embedding \
|
| 1511 |
+
--model.future-progress-dim=5 \
|
| 1512 |
+
--model.rtc \
|
| 1513 |
+
--model.max-delay=11 \
|
| 1514 |
+
--train.loss-group-weight.joint-abs=1.0 \
|
| 1515 |
+
--train.loss-group-weight.root-delta-xy=3.0 \
|
| 1516 |
+
--train.loss-group-weight.root-rot5-delta=10.0 \
|
| 1517 |
+
--train.loss-group-weight.root-height=15.0 \
|
| 1518 |
+
--train.loss-group-weight.foot-contact=0.1"
|
| 1519 |
+
|
| 1520 |
+
torchrun --standalone \
|
| 1521 |
+
--local_addr="$TORCHRUN_LOCAL_ADDR" \
|
| 1522 |
+
--nproc_per_node="$NPROC_PER_NODE" \
|
| 1523 |
+
scripts/train.py $args "$@"
|
posttrain-v1-psi0-all-7task-cached-oss.sh
ADDED
|
@@ -0,0 +1,775 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#!/usr/bin/env bash
|
| 2 |
+
set -eo pipefail
|
| 3 |
+
|
| 4 |
+
PROJECT_ROOT=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
|
| 5 |
+
cd "$PROJECT_ROOT"
|
| 6 |
+
export PYTHONPATH="$PROJECT_ROOT/src${PYTHONPATH:+:$PYTHONPATH}"
|
| 7 |
+
export CUDA_VISIBLE_DEVICES=${CUDA_VISIBLE_DEVICES:-0,1,2,3,4,5,6,7}
|
| 8 |
+
export PSI_OUTPUT_DIR=${PSI_OUTPUT_DIR:-$PROJECT_ROOT/runs}
|
| 9 |
+
export PSI_LOG_REPORT_TO=${PSI_LOG_REPORT_TO:-wandb}
|
| 10 |
+
|
| 11 |
+
source "$PROJECT_ROOT/scripts/activate_psi_env.sh"
|
| 12 |
+
PSI_ENV_FILE=${PSI_ENV_FILE:-$PROJECT_ROOT/.env}
|
| 13 |
+
if [ -f "$PSI_ENV_FILE" ]; then
|
| 14 |
+
set -a
|
| 15 |
+
source "$PSI_ENV_FILE"
|
| 16 |
+
set +a
|
| 17 |
+
fi
|
| 18 |
+
|
| 19 |
+
verify_wandb_auth() {
|
| 20 |
+
if [ "${PSI_LOG_REPORT_TO:-}" != "wandb" ]; then
|
| 21 |
+
return
|
| 22 |
+
fi
|
| 23 |
+
if [ -z "${WANDB_API_KEY:-}" ]; then
|
| 24 |
+
echo "[error] WandB logging is enabled but WANDB_API_KEY is empty." >&2
|
| 25 |
+
echo " Put WANDB_API_KEY in $PROJECT_ROOT/.env or export it before launch." >&2
|
| 26 |
+
exit 2
|
| 27 |
+
fi
|
| 28 |
+
python - <<'PY'
|
| 29 |
+
import os
|
| 30 |
+
import sys
|
| 31 |
+
|
| 32 |
+
try:
|
| 33 |
+
import wandb
|
| 34 |
+
|
| 35 |
+
timeout = int(os.environ.get("PSI_WANDB_AUTH_TIMEOUT", "30"))
|
| 36 |
+
viewer = wandb.Api(timeout=timeout).viewer
|
| 37 |
+
if viewer is None:
|
| 38 |
+
raise RuntimeError("WandB returned no authenticated user")
|
| 39 |
+
except Exception as exc:
|
| 40 |
+
print(
|
| 41 |
+
f"[error] WandB authentication failed ({type(exc).__name__}): {exc}",
|
| 42 |
+
file=sys.stderr,
|
| 43 |
+
)
|
| 44 |
+
sys.exit(2)
|
| 45 |
+
|
| 46 |
+
print("[ready] WandB authentication verified before data preflight.")
|
| 47 |
+
PY
|
| 48 |
+
}
|
| 49 |
+
|
| 50 |
+
# Fail in seconds, before copying multi-GiB indexes or constructing the model.
|
| 51 |
+
verify_wandb_auth
|
| 52 |
+
|
| 53 |
+
export OMP_NUM_THREADS=8
|
| 54 |
+
export TF_CPP_MIN_LOG_LEVEL=3
|
| 55 |
+
export PYTORCH_CUDA_ALLOC_CONF=${PYTORCH_CUDA_ALLOC_CONF:-expandable_segments:True}
|
| 56 |
+
export PSI_FLOW_SIGMA_MIN=${PSI_FLOW_SIGMA_MIN:-0.05}
|
| 57 |
+
export PSI_FLOW_SIGMA_MAX=${PSI_FLOW_SIGMA_MAX:-1.0}
|
| 58 |
+
export PSI_SKIP_VLM_MODEL=${PSI_SKIP_VLM_MODEL:-1}
|
| 59 |
+
export PSI_TASK_MIXTURE_SYNC_RANKS=${PSI_TASK_MIXTURE_SYNC_RANKS:-0}
|
| 60 |
+
export PSI_TASK_MIXTURE_SHARD_AWARE=${PSI_TASK_MIXTURE_SHARD_AWARE:-1}
|
| 61 |
+
export PSI_TASK_MIXTURE_RANK_LOCAL_BLOCKS=${PSI_TASK_MIXTURE_RANK_LOCAL_BLOCKS:-1}
|
| 62 |
+
# Keep every global optimizer step at a fixed task composition. With the
|
| 63 |
+
# defaults below, eight ranks are split 3 text / 1 audio / 3 audio-command /
|
| 64 |
+
# 1 video-clip. Ratios that cannot be represented exactly by the world size
|
| 65 |
+
# fall back to the legacy ratio-aware schedule.
|
| 66 |
+
export PSI_TASK_MIXTURE_BALANCED_RANKS=${PSI_TASK_MIXTURE_BALANCED_RANKS:-1}
|
| 67 |
+
export PSI_OMNI_OSS_TRAINING=${PSI_OMNI_OSS_TRAINING:-1}
|
| 68 |
+
export PSI_OMNI_OSS_EVAL_SHARD_AWARE=${PSI_OMNI_OSS_EVAL_SHARD_AWARE:-1}
|
| 69 |
+
export PSI_OMNI_OSS_EVAL_FULL_SHARD_CACHE=${PSI_OMNI_OSS_EVAL_FULL_SHARD_CACHE:-0}
|
| 70 |
+
export PSI_OMNI_OSS_EVAL_PREFETCH_LEAD_STEPS=${PSI_OMNI_OSS_EVAL_PREFETCH_LEAD_STEPS:-768}
|
| 71 |
+
export PSI_OMNI_OSS_EVAL_LOCAL_STAGE=${PSI_OMNI_OSS_EVAL_LOCAL_STAGE:-1}
|
| 72 |
+
export PSI_OMNI_OSS_EVAL_LOCAL_STAGE_DIR=${PSI_OMNI_OSS_EVAL_LOCAL_STAGE_DIR:-/tmp/psi0_v26_omni_eval_shards}
|
| 73 |
+
export PSI_OMNI_OSS_EVAL_FIXED_BATCHES=${PSI_OMNI_OSS_EVAL_FIXED_BATCHES:-1}
|
| 74 |
+
export PSI_OMNI_OSS_EVAL_CPU_BATCH_CACHE=${PSI_OMNI_OSS_EVAL_CPU_BATCH_CACHE:-1}
|
| 75 |
+
export PSI_OMNI_OSS_EVAL_PREWARM=${PSI_OMNI_OSS_EVAL_PREWARM:-1}
|
| 76 |
+
export PSI_OMNI_OSS_EVAL_VIDEO_WORKERS=${PSI_OMNI_OSS_EVAL_VIDEO_WORKERS:-3}
|
| 77 |
+
AVAILABLE_RAM_GIB=$(awk '/MemAvailable:/ {print int($2 / 1024 / 1024); exit}' /proc/meminfo 2>/dev/null || echo 0)
|
| 78 |
+
if [ "$AVAILABLE_RAM_GIB" -ge 1024 ]; then
|
| 79 |
+
# Direct-OSS measurements found two workers per rank faster and much less
|
| 80 |
+
# bursty than four; reuse=6 keeps those two workers compute-fed.
|
| 81 |
+
DEFAULT_TRAIN_WORKERS=2
|
| 82 |
+
DEFAULT_VAL_WORKERS=2
|
| 83 |
+
DEFAULT_DATALOADER_PREFETCH_FACTOR=2
|
| 84 |
+
DEFAULT_MOTION_CACHE_SIZE=4096
|
| 85 |
+
DEFAULT_AUDIO_CACHE_SIZE=1024
|
| 86 |
+
DEFAULT_PIN_MEMORY=1
|
| 87 |
+
elif [ "$AVAILABLE_RAM_GIB" -ge 512 ]; then
|
| 88 |
+
DEFAULT_TRAIN_WORKERS=2
|
| 89 |
+
DEFAULT_VAL_WORKERS=2
|
| 90 |
+
DEFAULT_DATALOADER_PREFETCH_FACTOR=3
|
| 91 |
+
DEFAULT_MOTION_CACHE_SIZE=2048
|
| 92 |
+
DEFAULT_AUDIO_CACHE_SIZE=512
|
| 93 |
+
DEFAULT_PIN_MEMORY=1
|
| 94 |
+
else
|
| 95 |
+
DEFAULT_TRAIN_WORKERS=2
|
| 96 |
+
DEFAULT_VAL_WORKERS=2
|
| 97 |
+
DEFAULT_DATALOADER_PREFETCH_FACTOR=2
|
| 98 |
+
DEFAULT_MOTION_CACHE_SIZE=256
|
| 99 |
+
DEFAULT_AUDIO_CACHE_SIZE=128
|
| 100 |
+
DEFAULT_PIN_MEMORY=0
|
| 101 |
+
fi
|
| 102 |
+
# Keep complete OSS shards resident in each worker so consecutive shard-aware
|
| 103 |
+
# batches reuse RAM instead of repeating remote slice reads. Returned samples
|
| 104 |
+
# are cloned below the cache layer before DataLoader IPC.
|
| 105 |
+
export PSI_OMNI_FULL_SHARD_CACHE_SIZE=${PSI_OMNI_FULL_SHARD_CACHE_SIZE:-4}
|
| 106 |
+
export PSI_OMNI_FULL_SHARD_CACHE_MAX_GIB=${PSI_OMNI_FULL_SHARD_CACHE_MAX_GIB:-4}
|
| 107 |
+
export PSI_OMNI_FULL_SHARD_CACHE_LOG=${PSI_OMNI_FULL_SHARD_CACHE_LOG:-0}
|
| 108 |
+
export PSI_OMNI_FULL_SHARD_PREFETCH=${PSI_OMNI_FULL_SHARD_PREFETCH:-1}
|
| 109 |
+
export PSI_OMNI_FULL_SHARD_PREFETCH_STAGGER=${PSI_OMNI_FULL_SHARD_PREFETCH_STAGGER:-32}
|
| 110 |
+
# OSS throughput degrades under unconstrained worker concurrency. Bound large
|
| 111 |
+
# shard transfers to the measured two-download sweet spot; worker-local RAM
|
| 112 |
+
# still serves all foreground sample slices in parallel.
|
| 113 |
+
export PSI_OMNI_FULL_SHARD_IO_SLOTS=${PSI_OMNI_FULL_SHARD_IO_SLOTS:-2}
|
| 114 |
+
export PSI_OMNI_FULL_SHARD_IO_LOCK_DIR=${PSI_OMNI_FULL_SHARD_IO_LOCK_DIR:-/tmp/psi0_v26_omni_io_slots}
|
| 115 |
+
export PSI_OMNI_FULL_SHARD_IO_LOCK_LOG=${PSI_OMNI_FULL_SHARD_IO_LOCK_LOG:-0}
|
| 116 |
+
# Bypass the mounted OSS FUSE path for multi-GiB shard materialization. Two
|
| 117 |
+
# node-wide downloads, each using 32 ranged GETs, was the fastest stable setup
|
| 118 |
+
# in the four-H200 comparison. Each completed object is cloned into the
|
| 119 |
+
# worker's resident RAM cache and its scratch file is removed immediately.
|
| 120 |
+
export PSI_OMNI_DIRECT_OSS_STAGE=${PSI_OMNI_DIRECT_OSS_STAGE:-1}
|
| 121 |
+
export PSI_OMNI_DIRECT_OSS_MOUNT_ROOT=${PSI_OMNI_DIRECT_OSS_MOUNT_ROOT:-/mnt/data}
|
| 122 |
+
export PSI_OMNI_DIRECT_OSS_URI_ROOT=${PSI_OMNI_DIRECT_OSS_URI_ROOT:-oss://cpfs-to-new-cpfs/VideoGen/user_data/buwendong}
|
| 123 |
+
export PSI_OMNI_DIRECT_OSS_STAGE_DIR=${PSI_OMNI_DIRECT_OSS_STAGE_DIR:-/tmp/psi0_v26_omni_direct_oss}
|
| 124 |
+
export PSI_OMNI_OSSUTIL_BIN=${PSI_OMNI_OSSUTIL_BIN:-/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/packages/ossutil-2.2.0-linux-amd64/ossutil}
|
| 125 |
+
export PSI_OMNI_OSSUTIL_CONFIG=${PSI_OMNI_OSSUTIL_CONFIG:-/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/packages/ossutil-2.2.0-linux-amd64/.ossutilconfig}
|
| 126 |
+
export PSI_OMNI_OSSUTIL_PARALLEL=${PSI_OMNI_OSSUTIL_PARALLEL:-32}
|
| 127 |
+
export PSI_OMNI_DIRECT_OSS_STAGE_LOG=${PSI_OMNI_DIRECT_OSS_STAGE_LOG:-0}
|
| 128 |
+
# Video-clip records reference full source motions (three CSVs per motion).
|
| 129 |
+
# Build exact float32 arrays in node-local storage and stage only the next
|
| 130 |
+
# shard's source motions while the current hidden-state shard is reused.
|
| 131 |
+
export PSI_VIDEO_MOTION_LOCAL_CACHE=${PSI_VIDEO_MOTION_LOCAL_CACHE:-1}
|
| 132 |
+
export PSI_VIDEO_MOTION_LOCAL_CACHE_DIR=${PSI_VIDEO_MOTION_LOCAL_CACHE_DIR:-/tmp/psi0_v26_video_motion_arrays}
|
| 133 |
+
export PSI_VIDEO_MOTION_PREFETCH=${PSI_VIDEO_MOTION_PREFETCH:-1}
|
| 134 |
+
export PSI_VIDEO_MOTION_PREFETCH_SLOTS=${PSI_VIDEO_MOTION_PREFETCH_SLOTS:-4}
|
| 135 |
+
export PSI_VIDEO_MOTION_PREFETCH_LOCK_DIR=${PSI_VIDEO_MOTION_PREFETCH_LOCK_DIR:-/tmp/psi0_v26_video_motion_prefetch_slots}
|
| 136 |
+
export PSI_VIDEO_MOTION_PREFETCH_LOG=${PSI_VIDEO_MOTION_PREFETCH_LOG:-0}
|
| 137 |
+
# Revisit each shuffled resident shard several times before replacement. With
|
| 138 |
+
# eight interleaved worker lanes, repeats are separated by a complete ~29k-
|
| 139 |
+
# sample lane pool rather than occurring in consecutive batches. This lowers
|
| 140 |
+
# required OSS bandwidth while preserving task weights and per-sample marginals.
|
| 141 |
+
export PSI_OMNI_OSS_SHARD_REUSE_PASSES=${PSI_OMNI_OSS_SHARD_REUSE_PASSES:-6}
|
| 142 |
+
export PSI_OMNI_HIDDEN_TENSOR_CACHE_SIZE=${PSI_OMNI_HIDDEN_TENSOR_CACHE_SIZE:-0}
|
| 143 |
+
export PSI_OMNI_SAFE_OPEN_CACHE_SIZE=${PSI_OMNI_SAFE_OPEN_CACHE_SIZE:-1}
|
| 144 |
+
export PSI_FOREACH_GRAD_NORM=${PSI_FOREACH_GRAD_NORM:-1}
|
| 145 |
+
export PSI_DATALOADER_PREFETCH_FACTOR=${PSI_DATALOADER_PREFETCH_FACTOR:-$DEFAULT_DATALOADER_PREFETCH_FACTOR}
|
| 146 |
+
export PSI_DATALOADER_PIN_MEMORY=${PSI_DATALOADER_PIN_MEMORY:-$DEFAULT_PIN_MEMORY}
|
| 147 |
+
export PSI_WORKER_TORCH_THREADS=${PSI_WORKER_TORCH_THREADS:-1}
|
| 148 |
+
export PSI_VIDEO_UNIQUE_RECORD_CACHE_FALLBACK=${PSI_VIDEO_UNIQUE_RECORD_CACHE_FALLBACK:-1}
|
| 149 |
+
export PSI_REQUIRE_PREBUILT_VIDEO_RECORD_CACHE=${PSI_REQUIRE_PREBUILT_VIDEO_RECORD_CACHE:-1}
|
| 150 |
+
export PSI_SOURCE_SPLIT_PREFLIGHT_AUDIT=${PSI_SOURCE_SPLIT_PREFLIGHT_AUDIT:-1}
|
| 151 |
+
TEXT_MOTION_CACHE_SIZE=${TEXT_MOTION_CACHE_SIZE:-$DEFAULT_MOTION_CACHE_SIZE}
|
| 152 |
+
AUDIO_MOTION_CACHE_SIZE=${AUDIO_MOTION_CACHE_SIZE:-$DEFAULT_MOTION_CACHE_SIZE}
|
| 153 |
+
AUDIO_COMMAND_MOTION_CACHE_SIZE=${AUDIO_COMMAND_MOTION_CACHE_SIZE:-$DEFAULT_MOTION_CACHE_SIZE}
|
| 154 |
+
AUDIO_ARRAY_CACHE_SIZE=${AUDIO_ARRAY_CACHE_SIZE:-$DEFAULT_AUDIO_CACHE_SIZE}
|
| 155 |
+
VIDEO_CLIP_MOTION_CACHE_SIZE=${VIDEO_CLIP_MOTION_CACHE_SIZE:-$DEFAULT_MOTION_CACHE_SIZE}
|
| 156 |
+
PSI_OMNI_OSS_ROOT=${PSI_OMNI_HIDDEN_CACHE_ROOT:-/mnt/data/data/omni_hidden_states_v23_oss}
|
| 157 |
+
PSI_OMNI_OSS_TRAINING_SUPPORT_ROOT=${PSI_OMNI_OSS_TRAINING_SUPPORT_ROOT:-$PSI_OMNI_OSS_ROOT/training_support}
|
| 158 |
+
# OSS remains the only persistent copy. Each launcher prefers one shared tmpfs
|
| 159 |
+
# copy so all local ranks and DataLoader workers read RAM rather than repeatedly
|
| 160 |
+
# seeking through the mounted OSS file. Node-local /tmp is the safe fallback.
|
| 161 |
+
PSI_SOURCE_SPLIT_OSS_ROOT=${PSI_SOURCE_SPLIT_OSS_ROOT:-$PSI_OMNI_OSS_TRAINING_SUPPORT_ROOT/source_split_v1}
|
| 162 |
+
if [ -z "${PSI_SOURCE_SPLIT_EPHEMERAL_ROOT:-}" ]; then
|
| 163 |
+
PSI_SOURCE_SPLIT_EPHEMERAL_PARENT=$(python - "$PSI_SOURCE_SPLIT_OSS_ROOT" <<'PY'
|
| 164 |
+
import json
|
| 165 |
+
import shutil
|
| 166 |
+
import sys
|
| 167 |
+
from pathlib import Path
|
| 168 |
+
|
| 169 |
+
source_root = Path(sys.argv[1])
|
| 170 |
+
manifest = json.loads((source_root / "artifact_manifest.json").read_text(encoding="utf-8"))
|
| 171 |
+
required = int(manifest["total_bytes"]) + 1024**3
|
| 172 |
+
for candidate in (Path("/dev/shm"), Path("/tmp")):
|
| 173 |
+
if candidate.is_dir() and shutil.disk_usage(candidate).free >= required:
|
| 174 |
+
print(candidate)
|
| 175 |
+
break
|
| 176 |
+
else:
|
| 177 |
+
raise SystemExit(
|
| 178 |
+
f"Neither /dev/shm nor /tmp has {required / 1024**3:.3f} GiB free "
|
| 179 |
+
"for source-split staging"
|
| 180 |
+
)
|
| 181 |
+
PY
|
| 182 |
+
)
|
| 183 |
+
PSI_SOURCE_SPLIT_EPHEMERAL_ROOT=$PSI_SOURCE_SPLIT_EPHEMERAL_PARENT/psi0_v26_source_split_v1.$$
|
| 184 |
+
fi
|
| 185 |
+
PSI_SOURCE_SPLIT_ROOT=$PSI_SOURCE_SPLIT_EPHEMERAL_ROOT
|
| 186 |
+
PSI_SOURCE_SPLIT_EPHEMERAL_CLEANUP=${PSI_SOURCE_SPLIT_EPHEMERAL_CLEANUP:-1}
|
| 187 |
+
export PSI_OMNI_OSS_RUNTIME_INDEX_DIR=$PSI_SOURCE_SPLIT_ROOT/runtime_indexes
|
| 188 |
+
PSI_SOURCE_SPLIT_AUDIT_REPORT=${PSI_SOURCE_SPLIT_AUDIT_REPORT:-$PSI_SOURCE_SPLIT_EPHEMERAL_ROOT/preflight_audit_report.json}
|
| 189 |
+
export PSI_OMNI_SQLITE_LOCAL_CACHE=${PSI_OMNI_SQLITE_LOCAL_CACHE:-1}
|
| 190 |
+
export PSI_OMNI_SQLITE_LOCAL_CACHE_DIR=${PSI_OMNI_SQLITE_LOCAL_CACHE_DIR:-/tmp/psi0_v26_omni_sqlite_indexes}
|
| 191 |
+
# The verified SQLite recovery snapshot is stored locally under this checkout;
|
| 192 |
+
# the large ignored files are never added to Git.
|
| 193 |
+
export PSI_OMNI_SQLITE_FALLBACK_ROOT=${PSI_OMNI_SQLITE_FALLBACK_ROOT:-$PROJECT_ROOT/recovery/omni_task_sqlite_20260716}
|
| 194 |
+
export PSI_OMNI_SQLITE_LOCAL_CACHE_LOCK_TIMEOUT=${PSI_OMNI_SQLITE_LOCAL_CACHE_LOCK_TIMEOUT:-1800}
|
| 195 |
+
# The copied lazy video-record caches contain absolute paths from their source
|
| 196 |
+
# tree. Rebase only that exact source prefix at read time so the existing
|
| 197 |
+
# offsets/sample indexes remain reusable while motion/video reads use OSS.
|
| 198 |
+
export PSI_VIDEO_RECORD_PATH_REWRITE_FROM=${PSI_VIDEO_RECORD_PATH_REWRITE_FROM:-/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/videogen/omni_hidden_states_v23_oss}
|
| 199 |
+
export PSI_VIDEO_RECORD_PATH_REWRITE_TO=${PSI_VIDEO_RECORD_PATH_REWRITE_TO:-$PSI_OMNI_OSS_ROOT}
|
| 200 |
+
|
| 201 |
+
# Keep JSON sources readable in this script, but pass compact JSON to tyro.
|
| 202 |
+
compact_json() {
|
| 203 |
+
python -c 'import json, sys; print(json.dumps(json.load(sys.stdin), separators=(",", ":")))'
|
| 204 |
+
}
|
| 205 |
+
|
| 206 |
+
PSI_SOURCE_SPLIT_EPHEMERAL_OWNED=0
|
| 207 |
+
cleanup_source_split_ephemeral() {
|
| 208 |
+
local exit_code=$?
|
| 209 |
+
trap - EXIT
|
| 210 |
+
if [ "$PSI_SOURCE_SPLIT_EPHEMERAL_OWNED" = "1" ] && \
|
| 211 |
+
{ [ "$PSI_SOURCE_SPLIT_EPHEMERAL_CLEANUP" = "1" ] || [ "$PSI_SOURCE_SPLIT_EPHEMERAL_CLEANUP" = "true" ]; }; then
|
| 212 |
+
if ! python -u scripts/data/manage_source_split_artifacts.py cleanup-ephemeral \
|
| 213 |
+
--root "$PSI_SOURCE_SPLIT_EPHEMERAL_ROOT"; then
|
| 214 |
+
echo "[warning] Failed to clean source-split ephemeral staging: $PSI_SOURCE_SPLIT_EPHEMERAL_ROOT" >&2
|
| 215 |
+
fi
|
| 216 |
+
fi
|
| 217 |
+
exit "$exit_code"
|
| 218 |
+
}
|
| 219 |
+
trap cleanup_source_split_ephemeral EXIT
|
| 220 |
+
|
| 221 |
+
PSI_SEED_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/Psi0/dataset/seed_all
|
| 222 |
+
PSI_SEED_METADATA=metadata/seed_metadata_v004.csv
|
| 223 |
+
PSI_TEXT_RECORDS_CACHE_DIR=${PSI_TEXT_RECORDS_CACHE_DIR:-$PSI_SOURCE_SPLIT_ROOT/text_records}
|
| 224 |
+
|
| 225 |
+
PSI_DANCE_MOTION_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/GR00T-WholeBodyControl/gear_sonic_deploy/reference/converted/psi0_dance/gvhmr
|
| 226 |
+
PSI_DANCE_AUDIO_VIDEO_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/dance_1723_video_psi0/dance_1723_video_10s
|
| 227 |
+
PSI_DANCE_VIDEO_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/dance_1723_video_psi0/dance_1723_video_10s
|
| 228 |
+
PSI_DANCE_VIDEO_EMBED_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/dance_1723_video_psi0/dance_1723_video_10s_embeds
|
| 229 |
+
PSI_VIDEO_DEMO_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/videogen/motion_example_0506_demo
|
| 230 |
+
PSI_2264_WITH_AUDIO_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/videogen/2264_with_audio
|
| 231 |
+
PSI_QWEN3_TTS_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/qwen3-tts
|
| 232 |
+
PSI_VIDEO_CLIP_DATASET_ROOT=${PSI_VIDEO_CLIP_DATASET_ROOT:-$PSI_OMNI_OSS_TRAINING_SUPPORT_ROOT/video_clip_dense_stride1}
|
| 233 |
+
PSI_DANCE_VIDEO_CLIP_ROOT=$PSI_VIDEO_CLIP_DATASET_ROOT/dance_1723_video_10s
|
| 234 |
+
PSI_MOTION_EXAMPLE_VIDEO_CLIP_ROOT=$PSI_VIDEO_CLIP_DATASET_ROOT/motion_example_0506_demo
|
| 235 |
+
PSI_2264_WITH_AUDIO_VIDEO_CLIP_ROOT=$PSI_VIDEO_CLIP_DATASET_ROOT/2264_with_audio
|
| 236 |
+
PSI_DANCE_VIDEO_CLIP_EMBED_ROOT=${PSI_DANCE_VIDEO_CLIP_EMBED_ROOT:-$PSI_DANCE_VIDEO_CLIP_ROOT/embeds}
|
| 237 |
+
PSI_MOTION_EXAMPLE_VIDEO_CLIP_EMBED_ROOT=${PSI_MOTION_EXAMPLE_VIDEO_CLIP_EMBED_ROOT:-$PSI_MOTION_EXAMPLE_VIDEO_CLIP_ROOT/embeds}
|
| 238 |
+
PSI_2264_WITH_AUDIO_VIDEO_CLIP_EMBED_ROOT=${PSI_2264_WITH_AUDIO_VIDEO_CLIP_EMBED_ROOT:-$PSI_2264_WITH_AUDIO_VIDEO_CLIP_ROOT/embeds}
|
| 239 |
+
PSI_MOTION_EXAMPLE_VIDEO_EMBED_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/videogen/motion_example_0506_demo_video_embeds
|
| 240 |
+
PSI_AUDIO_RECORDS_CACHE_DIR=${PSI_AUDIO_RECORDS_CACHE_DIR:-$PSI_SOURCE_SPLIT_ROOT/audio_records}
|
| 241 |
+
PSI_AUDIO_COMMAND_RECORDS_CACHE_DIR=${PSI_AUDIO_COMMAND_RECORDS_CACHE_DIR:-$PSI_SOURCE_SPLIT_ROOT/audio_command_records}
|
| 242 |
+
PSI_DANCE_AUDIO_NPY_ROOT=${PSI_DANCE_AUDIO_NPY_ROOT:-$PSI_DANCE_AUDIO_VIDEO_ROOT/audio_npy_yth}
|
| 243 |
+
PSI_DANCE_AUDIO_NPY_MANIFEST=${PSI_DANCE_AUDIO_NPY_MANIFEST:-$PSI_DANCE_AUDIO_VIDEO_ROOT/metadata_audio_npy.jsonl}
|
| 244 |
+
PSI_2264_AUDIO_NPY_ROOT=${PSI_2264_AUDIO_NPY_ROOT:-$PSI_2264_WITH_AUDIO_ROOT/audio_npy_yth}
|
| 245 |
+
PSI_2264_AUDIO_NPY_MANIFEST=${PSI_2264_AUDIO_NPY_MANIFEST:-$PSI_2264_WITH_AUDIO_ROOT/metadata_audio_npy.jsonl}
|
| 246 |
+
PSI_QWEN3_TTS_AUDIO_NPY_ROOT=${PSI_QWEN3_TTS_AUDIO_NPY_ROOT:-$PSI_QWEN3_TTS_ROOT/audio_npy_yth}
|
| 247 |
+
PSI_QWEN3_TTS_AUDIO_NPY_MANIFEST=${PSI_QWEN3_TTS_AUDIO_NPY_MANIFEST:-$PSI_QWEN3_TTS_ROOT/metadata_audio_npy.jsonl}
|
| 248 |
+
PSI_OMNI_HIDDEN_CACHE_ROOT=${PSI_OMNI_HIDDEN_CACHE_ROOT:-$PSI_OMNI_OSS_ROOT}
|
| 249 |
+
|
| 250 |
+
if [ -f "$PSI_2264_AUDIO_NPY_MANIFEST" ]; then
|
| 251 |
+
PSI_2264_AUDIO_METADATA_JSONL=$PSI_2264_AUDIO_NPY_MANIFEST
|
| 252 |
+
PSI_2264_AUDIO_PATH_KEY=audio_path
|
| 253 |
+
else
|
| 254 |
+
PSI_2264_AUDIO_METADATA_JSONL=metadata_convert.jsonl
|
| 255 |
+
PSI_2264_AUDIO_PATH_KEY=video_path
|
| 256 |
+
fi
|
| 257 |
+
|
| 258 |
+
if [ -f "$PSI_DANCE_AUDIO_NPY_MANIFEST" ]; then
|
| 259 |
+
PSI_DANCE_AUDIO_SOURCE_JSON=$(cat <<JSON
|
| 260 |
+
{
|
| 261 |
+
"name": "psi0_dance_audio",
|
| 262 |
+
"split_source_namespace": "dance_1723_video_10s",
|
| 263 |
+
"dataset_root": "$PSI_DANCE_AUDIO_VIDEO_ROOT",
|
| 264 |
+
"metadata_jsonl": "$PSI_DANCE_AUDIO_NPY_MANIFEST",
|
| 265 |
+
"motion_dir_key": "converted_path",
|
| 266 |
+
"audio_path_key": "audio_path",
|
| 267 |
+
"records_cache_dir": "$PSI_AUDIO_RECORDS_CACHE_DIR",
|
| 268 |
+
"motion_cache_size": $AUDIO_MOTION_CACHE_SIZE,
|
| 269 |
+
"audio_cache_size": $AUDIO_ARRAY_CACHE_SIZE,
|
| 270 |
+
"dataset_name": "seed-audio-g1"
|
| 271 |
+
}
|
| 272 |
+
JSON
|
| 273 |
+
)
|
| 274 |
+
else
|
| 275 |
+
PSI_DANCE_AUDIO_SOURCE_JSON=$(cat <<JSON
|
| 276 |
+
{
|
| 277 |
+
"name": "psi0_dance_audio",
|
| 278 |
+
"split_source_namespace": "dance_1723_video_10s",
|
| 279 |
+
"root_dir": "$PSI_DANCE_MOTION_ROOT",
|
| 280 |
+
"video_dir": "$PSI_DANCE_AUDIO_VIDEO_ROOT",
|
| 281 |
+
"records_cache_dir": "$PSI_AUDIO_RECORDS_CACHE_DIR",
|
| 282 |
+
"dataset_name": "seed-audio-g1"
|
| 283 |
+
}
|
| 284 |
+
JSON
|
| 285 |
+
)
|
| 286 |
+
fi
|
| 287 |
+
|
| 288 |
+
if [ -z "$PSI_AUDIO_SOURCES_JSON" ]; then
|
| 289 |
+
PSI_AUDIO_SOURCES_JSON=$(compact_json <<JSON
|
| 290 |
+
[
|
| 291 |
+
$PSI_DANCE_AUDIO_SOURCE_JSON,
|
| 292 |
+
{
|
| 293 |
+
"name": "2264_with_audio",
|
| 294 |
+
"split_source_namespace": "2264_with_audio",
|
| 295 |
+
"dataset_root": "$PSI_2264_WITH_AUDIO_ROOT",
|
| 296 |
+
"metadata_jsonl": "$PSI_2264_AUDIO_METADATA_JSONL",
|
| 297 |
+
"motion_dir_key": "converted_path",
|
| 298 |
+
"audio_path_key": "$PSI_2264_AUDIO_PATH_KEY",
|
| 299 |
+
"records_cache_dir": "$PSI_AUDIO_RECORDS_CACHE_DIR",
|
| 300 |
+
"motion_cache_size": $AUDIO_MOTION_CACHE_SIZE,
|
| 301 |
+
"audio_cache_size": $AUDIO_ARRAY_CACHE_SIZE,
|
| 302 |
+
"dataset_name": "2264-with-audio-g1"
|
| 303 |
+
}
|
| 304 |
+
]
|
| 305 |
+
JSON
|
| 306 |
+
)
|
| 307 |
+
fi
|
| 308 |
+
|
| 309 |
+
if [ -f "$PSI_QWEN3_TTS_AUDIO_NPY_MANIFEST" ]; then
|
| 310 |
+
PSI_AUDIO_COMMAND_SOURCE_JSON=$(cat <<JSON
|
| 311 |
+
{
|
| 312 |
+
"name": "qwen3_tts_seed_audio_command",
|
| 313 |
+
"split_source_namespace": "seed",
|
| 314 |
+
"root_dir": "$PSI_SEED_ROOT",
|
| 315 |
+
"dataset_root": "$PSI_QWEN3_TTS_ROOT",
|
| 316 |
+
"metadata_jsonl": "$PSI_QWEN3_TTS_AUDIO_NPY_MANIFEST",
|
| 317 |
+
"motion_dir_key": "filename",
|
| 318 |
+
"audio_path_key": "audio_path",
|
| 319 |
+
"instruction_key": "content_short_description",
|
| 320 |
+
"motion_metadata_csv": "$PSI_SEED_METADATA",
|
| 321 |
+
"motion_metadata_key": "filename",
|
| 322 |
+
"motion_metadata_path_key": "move_g1_path",
|
| 323 |
+
"audio_file_extensions": ".npy",
|
| 324 |
+
"records_cache_dir": "$PSI_AUDIO_COMMAND_RECORDS_CACHE_DIR",
|
| 325 |
+
"motion_cache_size": $AUDIO_COMMAND_MOTION_CACHE_SIZE,
|
| 326 |
+
"audio_cache_size": $AUDIO_ARRAY_CACHE_SIZE,
|
| 327 |
+
"audio_task": "command",
|
| 328 |
+
"dataset_name": "seed-audio-command-g1"
|
| 329 |
+
}
|
| 330 |
+
JSON
|
| 331 |
+
)
|
| 332 |
+
else
|
| 333 |
+
PSI_AUDIO_COMMAND_SOURCE_JSON=$(cat <<JSON
|
| 334 |
+
{
|
| 335 |
+
"name": "qwen3_tts_seed_audio_command",
|
| 336 |
+
"split_source_namespace": "seed",
|
| 337 |
+
"root_dir": "$PSI_SEED_ROOT",
|
| 338 |
+
"dataset_root": "$PSI_QWEN3_TTS_ROOT",
|
| 339 |
+
"metadata_jsonl": "metadata.jsonl",
|
| 340 |
+
"motion_dir_key": "filename",
|
| 341 |
+
"audio_path_key": "audio_path",
|
| 342 |
+
"instruction_key": "content_short_description",
|
| 343 |
+
"motion_metadata_csv": "$PSI_SEED_METADATA",
|
| 344 |
+
"motion_metadata_key": "filename",
|
| 345 |
+
"motion_metadata_path_key": "move_g1_path",
|
| 346 |
+
"audio_file_extensions": ".wav",
|
| 347 |
+
"records_cache_dir": "$PSI_AUDIO_COMMAND_RECORDS_CACHE_DIR",
|
| 348 |
+
"motion_cache_size": $AUDIO_COMMAND_MOTION_CACHE_SIZE,
|
| 349 |
+
"audio_cache_size": $AUDIO_ARRAY_CACHE_SIZE,
|
| 350 |
+
"audio_task": "command",
|
| 351 |
+
"dataset_name": "seed-audio-command-g1"
|
| 352 |
+
}
|
| 353 |
+
JSON
|
| 354 |
+
)
|
| 355 |
+
fi
|
| 356 |
+
|
| 357 |
+
if [ -z "$PSI_AUDIO_COMMAND_SOURCES_JSON" ]; then
|
| 358 |
+
PSI_AUDIO_COMMAND_SOURCES_JSON=$(compact_json <<JSON
|
| 359 |
+
[
|
| 360 |
+
$PSI_AUDIO_COMMAND_SOURCE_JSON
|
| 361 |
+
]
|
| 362 |
+
JSON
|
| 363 |
+
)
|
| 364 |
+
fi
|
| 365 |
+
|
| 366 |
+
if [ -z "$PSI_VIDEO_SOURCES_JSON" ]; then
|
| 367 |
+
PSI_VIDEO_SOURCES_JSON=$(compact_json <<JSON
|
| 368 |
+
[
|
| 369 |
+
{
|
| 370 |
+
"name": "psi0_dance_gvhmr",
|
| 371 |
+
"root_dir": "$PSI_DANCE_MOTION_ROOT",
|
| 372 |
+
"video_dir": "$PSI_DANCE_VIDEO_ROOT",
|
| 373 |
+
"video_embed_dir": "$PSI_DANCE_VIDEO_EMBED_ROOT",
|
| 374 |
+
"dataset_name": "seed-video-g1"
|
| 375 |
+
},
|
| 376 |
+
{
|
| 377 |
+
"name": "motion_example_0506_demo",
|
| 378 |
+
"dataset_root": "$PSI_VIDEO_DEMO_ROOT",
|
| 379 |
+
"metadata_jsonl": "metadata.jsonl",
|
| 380 |
+
"motion_dir_key": "converted_path",
|
| 381 |
+
"video_path_key": "video_path",
|
| 382 |
+
"instruction_key": "video_caption",
|
| 383 |
+
"video_embed_dir": "$PSI_MOTION_EXAMPLE_VIDEO_EMBED_ROOT",
|
| 384 |
+
"dataset_name": "motion-example-video-g1"
|
| 385 |
+
}
|
| 386 |
+
]
|
| 387 |
+
JSON
|
| 388 |
+
)
|
| 389 |
+
fi
|
| 390 |
+
|
| 391 |
+
if [ -z "$PSI_VIDEO_CLIP_SOURCES_JSON" ]; then
|
| 392 |
+
PSI_VIDEO_CLIP_SOURCES_JSON=$(compact_json <<JSON
|
| 393 |
+
[
|
| 394 |
+
{
|
| 395 |
+
"name": "dance_1723_video_10s_clip50",
|
| 396 |
+
"split_source_namespace": "dance_1723_video_10s",
|
| 397 |
+
"dataset_root": "$PSI_DANCE_VIDEO_CLIP_ROOT",
|
| 398 |
+
"metadata_jsonl": "metadata.jsonl",
|
| 399 |
+
"motion_dir_key": "converted_path",
|
| 400 |
+
"video_path_key": "video_path",
|
| 401 |
+
"instruction_key": "video_caption",
|
| 402 |
+
"video_embed_dir": "$PSI_DANCE_VIDEO_CLIP_EMBED_ROOT",
|
| 403 |
+
"motion_cache_size": $VIDEO_CLIP_MOTION_CACHE_SIZE,
|
| 404 |
+
"dataset_name": "dance-video-clip-g1"
|
| 405 |
+
},
|
| 406 |
+
{
|
| 407 |
+
"name": "motion_example_0506_demo_clip50",
|
| 408 |
+
"split_source_namespace": "motion_example_0506_demo",
|
| 409 |
+
"dataset_root": "$PSI_MOTION_EXAMPLE_VIDEO_CLIP_ROOT",
|
| 410 |
+
"metadata_jsonl": "metadata.jsonl",
|
| 411 |
+
"motion_dir_key": "converted_path",
|
| 412 |
+
"video_path_key": "video_path",
|
| 413 |
+
"instruction_key": "video_caption",
|
| 414 |
+
"video_embed_dir": "$PSI_MOTION_EXAMPLE_VIDEO_CLIP_EMBED_ROOT",
|
| 415 |
+
"motion_cache_size": $VIDEO_CLIP_MOTION_CACHE_SIZE,
|
| 416 |
+
"dataset_name": "motion-example-video-clip-g1"
|
| 417 |
+
},
|
| 418 |
+
{
|
| 419 |
+
"name": "2264_with_audio_clip50",
|
| 420 |
+
"split_source_namespace": "2264_with_audio",
|
| 421 |
+
"dataset_root": "$PSI_2264_WITH_AUDIO_VIDEO_CLIP_ROOT",
|
| 422 |
+
"metadata_jsonl": "metadata.jsonl",
|
| 423 |
+
"motion_dir_key": "converted_path",
|
| 424 |
+
"video_path_key": "video_path",
|
| 425 |
+
"instruction_key": "video_caption",
|
| 426 |
+
"video_embed_dir": "$PSI_2264_WITH_AUDIO_VIDEO_CLIP_EMBED_ROOT",
|
| 427 |
+
"motion_cache_size": $VIDEO_CLIP_MOTION_CACHE_SIZE,
|
| 428 |
+
"dataset_name": "2264-with-audio-video-clip-g1"
|
| 429 |
+
}
|
| 430 |
+
]
|
| 431 |
+
JSON
|
| 432 |
+
)
|
| 433 |
+
fi
|
| 434 |
+
|
| 435 |
+
# Prompted-cache mix, excluding legacy VIDEO_TEXT / VIDEO_ONLY and all *_only
|
| 436 |
+
# tasks. The *_only cache directories are intentionally not required.
|
| 437 |
+
# Diagnostic task allocation for eight GPUs: text and audio-command each use
|
| 438 |
+
# three ranks, while the two modalities whose validation loss rose first
|
| 439 |
+
# (audio and video-clip) each use one rank. This changes their gradient weight
|
| 440 |
+
# from 25% each to 12.5% each while keeping every global step composition fixed.
|
| 441 |
+
# text stride1 frame samples: ~47,000,000
|
| 442 |
+
# music audio dance/audio frame samples use dense stride1 like video_clip
|
| 443 |
+
# audio_command wav files: 41,760
|
| 444 |
+
# dense video_clip metadata: 4,597,576
|
| 445 |
+
PSI_TEXT_TASK_WEIGHT=${PSI_TEXT_TASK_WEIGHT:-3}
|
| 446 |
+
PSI_AUDIO_TEXT_TASK_WEIGHT=${PSI_AUDIO_TEXT_TASK_WEIGHT:-1}
|
| 447 |
+
PSI_AUDIO_ONLY_TASK_WEIGHT=${PSI_AUDIO_ONLY_TASK_WEIGHT:-0}
|
| 448 |
+
PSI_AUDIO_COMMAND_TEXT_TASK_WEIGHT=${PSI_AUDIO_COMMAND_TEXT_TASK_WEIGHT:-3}
|
| 449 |
+
PSI_AUDIO_COMMAND_ONLY_TASK_WEIGHT=${PSI_AUDIO_COMMAND_ONLY_TASK_WEIGHT:-0}
|
| 450 |
+
PSI_VIDEO_TEXT_TASK_WEIGHT=${PSI_VIDEO_TEXT_TASK_WEIGHT:-0}
|
| 451 |
+
PSI_VIDEO_ONLY_TASK_WEIGHT=${PSI_VIDEO_ONLY_TASK_WEIGHT:-0}
|
| 452 |
+
PSI_VIDEO_CLIP_TEXT_TASK_WEIGHT=${PSI_VIDEO_CLIP_TEXT_TASK_WEIGHT:-1}
|
| 453 |
+
PSI_VIDEO_CLIP_ONLY_TASK_WEIGHT=${PSI_VIDEO_CLIP_ONLY_TASK_WEIGHT:-0}
|
| 454 |
+
|
| 455 |
+
PSI_PRETRAIN_BACKBONE_PATH=${PSI_PRETRAIN_BACKBONE_PATH:-/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/team_workspace/public_ckpt/Qwen/Qwen2.5-Omni-3B}
|
| 456 |
+
NUM_PAST_FRAMES=0
|
| 457 |
+
STATE_HORIZON=$((NUM_PAST_FRAMES + 1))
|
| 458 |
+
ACTION_CHUNK_SIZE=30
|
| 459 |
+
ACTION_EXEC_HORIZON=20
|
| 460 |
+
RTC_PREFIX_FRAMES=$((ACTION_CHUNK_SIZE - ACTION_EXEC_HORIZON))
|
| 461 |
+
if [ "$RTC_PREFIX_FRAMES" -lt 0 ] || [ "$RTC_PREFIX_FRAMES" -ge "$ACTION_CHUNK_SIZE" ]; then
|
| 462 |
+
echo "Invalid RTC prefix: chunk=$ACTION_CHUNK_SIZE exec=$ACTION_EXEC_HORIZON prefix=$RTC_PREFIX_FRAMES" >&2
|
| 463 |
+
exit 2
|
| 464 |
+
fi
|
| 465 |
+
PROGRESS_DIM=5
|
| 466 |
+
MOTION_FPS=50
|
| 467 |
+
AUDIO_WINDOW_SECONDS=$(awk "BEGIN {printf \"%.6f\", ${ACTION_CHUNK_SIZE} / ${MOTION_FPS}}")
|
| 468 |
+
AUDIO_SAMPLE_STEP_FRAMES=1
|
| 469 |
+
VIDEO_SAMPLE_FPS=1.0
|
| 470 |
+
VIDEO_CLIP_SAMPLE_FPS=8.0
|
| 471 |
+
VIDEO_SIZE_PIXELS=50176
|
| 472 |
+
VIDEO_DECODER=ffmpeg
|
| 473 |
+
VIDEO_CACHE_SIZE=64
|
| 474 |
+
VIDEO_CLIP_MOTION_CACHE_SIZE=${VIDEO_CLIP_MOTION_CACHE_SIZE:-$DEFAULT_MOTION_CACHE_SIZE}
|
| 475 |
+
VIDEO_RECORD_BUILD_WORKERS=${VIDEO_RECORD_BUILD_WORKERS:-32}
|
| 476 |
+
VIDEO_RECORD_BUILD_CHUNK_SIZE=${VIDEO_RECORD_BUILD_CHUNK_SIZE:-4096}
|
| 477 |
+
VIDEO_CLIP_TRUST_METADATA=${VIDEO_CLIP_TRUST_METADATA:-1}
|
| 478 |
+
VIDEO_CLIP_RECORDS_CACHE_DIR=${VIDEO_CLIP_RECORDS_CACHE_DIR:-$PSI_SOURCE_SPLIT_ROOT/video_clip_records}
|
| 479 |
+
if [ "$VIDEO_CLIP_TRUST_METADATA" = "1" ] || [ "$VIDEO_CLIP_TRUST_METADATA" = "true" ] || [ "$VIDEO_CLIP_TRUST_METADATA" = "True" ]; then
|
| 480 |
+
VIDEO_CLIP_TRUST_METADATA_FLAG="--data.video-clip.trust-metadata"
|
| 481 |
+
else
|
| 482 |
+
VIDEO_CLIP_TRUST_METADATA_FLAG="--data.video-clip.no-trust-metadata"
|
| 483 |
+
fi
|
| 484 |
+
|
| 485 |
+
# DATALOADER_NUM_WORKERS=4
|
| 486 |
+
# VAL_DATALOADER_NUM_WORKERS=2
|
| 487 |
+
|
| 488 |
+
# DATALOADER_NUM_WORKERS=${DATALOADER_NUM_WORKERS:-0}
|
| 489 |
+
# VAL_DATALOADER_NUM_WORKERS=${VAL_DATALOADER_NUM_WORKERS:-0}
|
| 490 |
+
|
| 491 |
+
DATALOADER_NUM_WORKERS=${DATALOADER_NUM_WORKERS:-$DEFAULT_TRAIN_WORKERS}
|
| 492 |
+
VAL_DATALOADER_NUM_WORKERS=${VAL_DATALOADER_NUM_WORKERS:-$DEFAULT_VAL_WORKERS}
|
| 493 |
+
TRAIN_BATCH_SIZE=${TRAIN_BATCH_SIZE:-64}
|
| 494 |
+
VAL_BATCH_SIZE=${VAL_BATCH_SIZE:-64}
|
| 495 |
+
# <modify>
|
| 496 |
+
GRADIENT_ACCUMULATION_STEPS=${GRADIENT_ACCUMULATION_STEPS:-1}
|
| 497 |
+
# <modify>
|
| 498 |
+
MAX_TRAINING_STEPS=${MAX_TRAINING_STEPS:-1000000}
|
| 499 |
+
WARMUP_STEPS=${WARMUP_STEPS:-1000}
|
| 500 |
+
TRAIN_SEED=${PSI_TRAIN_SEED:-292285}
|
| 501 |
+
CHECKPOINTING_STEPS=${CHECKPOINTING_STEPS:-1000}
|
| 502 |
+
VALIDATION_STEPS=${VALIDATION_STEPS:-1000}
|
| 503 |
+
VAL_NUM_BATCHES=${VAL_NUM_BATCHES:-20}
|
| 504 |
+
MASTER_PORT=${MASTER_PORT:-29500}
|
| 505 |
+
|
| 506 |
+
# Keep one independently cached shard per DataLoader worker lane. Each lane
|
| 507 |
+
# consumes roughly one train split worth of a 4096-sample object before the
|
| 508 |
+
# task schedule moves on. The block size is scaled automatically with the
|
| 509 |
+
# configured batch size and worker count.
|
| 510 |
+
OSS_ESTIMATED_TRAIN_SAMPLES_PER_SHARD=${OSS_ESTIMATED_TRAIN_SAMPLES_PER_SHARD:-3686}
|
| 511 |
+
if [ "$DATALOADER_NUM_WORKERS" -gt 0 ]; then
|
| 512 |
+
PSI_OMNI_OSS_WORKER_LANES=${PSI_OMNI_OSS_WORKER_LANES:-$DATALOADER_NUM_WORKERS}
|
| 513 |
+
else
|
| 514 |
+
PSI_OMNI_OSS_WORKER_LANES=${PSI_OMNI_OSS_WORKER_LANES:-1}
|
| 515 |
+
fi
|
| 516 |
+
OSS_BATCHES_PER_LANE=$(((OSS_ESTIMATED_TRAIN_SAMPLES_PER_SHARD + TRAIN_BATCH_SIZE - 1) / TRAIN_BATCH_SIZE))
|
| 517 |
+
export PSI_OMNI_OSS_WORKER_LANES
|
| 518 |
+
export PSI_TASK_MIXTURE_BLOCK_BATCHES=${PSI_TASK_MIXTURE_BLOCK_BATCHES:-$((OSS_BATCHES_PER_LANE * PSI_OMNI_OSS_WORKER_LANES))}
|
| 519 |
+
|
| 520 |
+
if [ ! -f "$PSI_PRETRAIN_BACKBONE_PATH/config.json" ]; then
|
| 521 |
+
echo "[error] Qwen backbone is missing or incomplete: $PSI_PRETRAIN_BACKBONE_PATH" >&2
|
| 522 |
+
exit 1
|
| 523 |
+
fi
|
| 524 |
+
|
| 525 |
+
case "${PSI_OMNI_DIRECT_OSS_STAGE,,}" in
|
| 526 |
+
1|true|yes|y|on)
|
| 527 |
+
if [ ! -x "$PSI_OMNI_OSSUTIL_BIN" ]; then
|
| 528 |
+
echo "[error] Direct OSS staging binary is missing or not executable: $PSI_OMNI_OSSUTIL_BIN" >&2
|
| 529 |
+
exit 1
|
| 530 |
+
fi
|
| 531 |
+
if [ ! -f "$PSI_OMNI_OSSUTIL_CONFIG" ]; then
|
| 532 |
+
echo "[error] Direct OSS staging config is missing: $PSI_OMNI_OSSUTIL_CONFIG" >&2
|
| 533 |
+
exit 1
|
| 534 |
+
fi
|
| 535 |
+
case "$PSI_OMNI_HIDDEN_CACHE_ROOT/" in
|
| 536 |
+
"$PSI_OMNI_DIRECT_OSS_MOUNT_ROOT"/*) ;;
|
| 537 |
+
*)
|
| 538 |
+
echo "[error] OSS hidden root is outside the direct-staging mount root:" >&2
|
| 539 |
+
echo " hidden=$PSI_OMNI_HIDDEN_CACHE_ROOT mount=$PSI_OMNI_DIRECT_OSS_MOUNT_ROOT" >&2
|
| 540 |
+
exit 1
|
| 541 |
+
;;
|
| 542 |
+
esac
|
| 543 |
+
;;
|
| 544 |
+
esac
|
| 545 |
+
|
| 546 |
+
PSI_SOURCE_SPLIT_EPHEMERAL_OWNED=1
|
| 547 |
+
python -u scripts/data/manage_source_split_artifacts.py stage \
|
| 548 |
+
--ephemeral \
|
| 549 |
+
--source "$PSI_SOURCE_SPLIT_OSS_ROOT" \
|
| 550 |
+
--destination "$PSI_SOURCE_SPLIT_EPHEMERAL_ROOT"
|
| 551 |
+
|
| 552 |
+
python -u scripts/data/prepare_omni_oss_training.py \
|
| 553 |
+
--oss-root "$PSI_OMNI_HIDDEN_CACHE_ROOT" \
|
| 554 |
+
--runtime-index-dir "$PSI_OMNI_OSS_RUNTIME_INDEX_DIR" \
|
| 555 |
+
--video-manifest-spec "dance_1723_video_10s=$PSI_DANCE_VIDEO_CLIP_ROOT/metadata.jsonl" \
|
| 556 |
+
--video-manifest-spec "motion_example_0506_demo=$PSI_MOTION_EXAMPLE_VIDEO_CLIP_ROOT/metadata.jsonl" \
|
| 557 |
+
--video-manifest-spec "2264_with_audio=$PSI_2264_WITH_AUDIO_VIDEO_CLIP_ROOT/metadata.jsonl"
|
| 558 |
+
|
| 559 |
+
if [ "$PSI_SOURCE_SPLIT_PREFLIGHT_AUDIT" = "1" ] || [ "$PSI_SOURCE_SPLIT_PREFLIGHT_AUDIT" = "true" ]; then
|
| 560 |
+
python -u scripts/data/audit_source_split_artifacts.py \
|
| 561 |
+
--skip-full-record-scan \
|
| 562 |
+
--cache-root "$PSI_SOURCE_SPLIT_ROOT" \
|
| 563 |
+
--runtime-index-dir "$PSI_OMNI_OSS_RUNTIME_INDEX_DIR" \
|
| 564 |
+
--report "$PSI_SOURCE_SPLIT_AUDIT_REPORT"
|
| 565 |
+
fi
|
| 566 |
+
|
| 567 |
+
if [ "${PSI_OMNI_OSS_PREFLIGHT_ONLY:-0}" = "1" ]; then
|
| 568 |
+
echo "[preflight-only] Validation complete; torchrun was intentionally skipped."
|
| 569 |
+
exit 0
|
| 570 |
+
fi
|
| 571 |
+
|
| 572 |
+
NPROC_PER_NODE=$(echo $CUDA_VISIBLE_DEVICES | tr ',' '\n' | wc -l)
|
| 573 |
+
EFFECTIVE_TRAIN_BATCH_SIZE=$((TRAIN_BATCH_SIZE * NPROC_PER_NODE * GRADIENT_ACCUMULATION_STEPS))
|
| 574 |
+
|
| 575 |
+
echo "Training with $NPROC_PER_NODE GPU(s): $CUDA_VISIBLE_DEVICES"
|
| 576 |
+
echo "Text root: $PSI_SEED_ROOT"
|
| 577 |
+
echo "Text records cache dir: $PSI_TEXT_RECORDS_CACHE_DIR"
|
| 578 |
+
echo "Source split persistent OSS root: $PSI_SOURCE_SPLIT_OSS_ROOT"
|
| 579 |
+
echo "Source split ephemeral root: $PSI_SOURCE_SPLIT_EPHEMERAL_ROOT"
|
| 580 |
+
echo "Dance motion root: $PSI_DANCE_MOTION_ROOT"
|
| 581 |
+
echo "Dance audio/video root: $PSI_DANCE_AUDIO_VIDEO_ROOT"
|
| 582 |
+
echo "Dance video root: $PSI_DANCE_VIDEO_ROOT"
|
| 583 |
+
echo "Dance video embed root: $PSI_DANCE_VIDEO_EMBED_ROOT"
|
| 584 |
+
echo "2264 with audio root: $PSI_2264_WITH_AUDIO_ROOT"
|
| 585 |
+
echo "Dance audio npy root: $PSI_DANCE_AUDIO_NPY_ROOT"
|
| 586 |
+
echo "Dance audio manifest: $PSI_DANCE_AUDIO_NPY_MANIFEST"
|
| 587 |
+
echo "2264 audio npy root: $PSI_2264_AUDIO_NPY_ROOT"
|
| 588 |
+
echo "2264 audio manifest: $PSI_2264_AUDIO_METADATA_JSONL"
|
| 589 |
+
echo "Audio records cache dir: $PSI_AUDIO_RECORDS_CACHE_DIR"
|
| 590 |
+
echo "Audio command records cache dir: $PSI_AUDIO_COMMAND_RECORDS_CACHE_DIR"
|
| 591 |
+
echo "Qwen3-TTS acdudio command root: $PSI_QWEN3_TTS_ROOT"
|
| 592 |
+
echo "Qwen3-TTS audio npy root: $PSI_QWEN3_TTS_AUDIO_NPY_ROOT"
|
| 593 |
+
echo "Qwen3-TTS audio manifest: $PSI_QWEN3_TTS_AUDIO_NPY_MANIFEST"
|
| 594 |
+
echo "Audio sources: $PSI_AUDIO_SOURCES_JSON"
|
| 595 |
+
echo "Audio command sources: $PSI_AUDIO_COMMAND_SOURCES_JSON"
|
| 596 |
+
echo "Video sources: $PSI_VIDEO_SOURCES_JSON"
|
| 597 |
+
echo "Video clip sources: $PSI_VIDEO_CLIP_SOURCES_JSON"
|
| 598 |
+
echo "DataLoader workers: train=$DATALOADER_NUM_WORKERS val=$VAL_DATALOADER_NUM_WORKERS"
|
| 599 |
+
echo "Available host RAM: ${AVAILABLE_RAM_GIB} GiB"
|
| 600 |
+
echo "DataLoader prefetch/pin: factor=$PSI_DATALOADER_PREFETCH_FACTOR pin=$PSI_DATALOADER_PIN_MEMORY worker_torch_threads=$PSI_WORKER_TORCH_THREADS"
|
| 601 |
+
echo "Motion/audio caches: text=$TEXT_MOTION_CACHE_SIZE audio=$AUDIO_MOTION_CACHE_SIZE audio_command=$AUDIO_COMMAND_MOTION_CACHE_SIZE audio_arrays=$AUDIO_ARRAY_CACHE_SIZE video_clip=$VIDEO_CLIP_MOTION_CACHE_SIZE"
|
| 602 |
+
echo "Sampler: task_mixture"
|
| 603 |
+
echo "Task mixture sync ranks: $PSI_TASK_MIXTURE_SYNC_RANKS"
|
| 604 |
+
echo "Task mixture shard aware: $PSI_TASK_MIXTURE_SHARD_AWARE"
|
| 605 |
+
echo "Task mixture local block batches: $PSI_TASK_MIXTURE_BLOCK_BATCHES"
|
| 606 |
+
echo "Task mixture balanced ranks: $PSI_TASK_MIXTURE_BALANCED_RANKS"
|
| 607 |
+
echo "OSS worker lanes: $PSI_OMNI_OSS_WORKER_LANES"
|
| 608 |
+
echo "OSS eval shard-aware batches: $PSI_OMNI_OSS_EVAL_SHARD_AWARE"
|
| 609 |
+
echo "OSS eval full-shard cache: $PSI_OMNI_OSS_EVAL_FULL_SHARD_CACHE"
|
| 610 |
+
echo "OSS eval CPU prefetch lead steps: $PSI_OMNI_OSS_EVAL_PREFETCH_LEAD_STEPS"
|
| 611 |
+
echo "OSS eval local stage: enabled=$PSI_OMNI_OSS_EVAL_LOCAL_STAGE dir=$PSI_OMNI_OSS_EVAL_LOCAL_STAGE_DIR"
|
| 612 |
+
echo "OSS eval fixed/CPU-cache/prewarm: fixed=$PSI_OMNI_OSS_EVAL_FIXED_BATCHES cache=$PSI_OMNI_OSS_EVAL_CPU_BATCH_CACHE prewarm=$PSI_OMNI_OSS_EVAL_PREWARM"
|
| 613 |
+
echo "OSS eval video workers: $PSI_OMNI_OSS_EVAL_VIDEO_WORKERS"
|
| 614 |
+
echo "OSS runtime index dir: $PSI_OMNI_OSS_RUNTIME_INDEX_DIR"
|
| 615 |
+
echo "Video record path rewrite: $PSI_VIDEO_RECORD_PATH_REWRITE_FROM -> $PSI_VIDEO_RECORD_PATH_REWRITE_TO"
|
| 616 |
+
echo "OSS local SQLite cache: enabled=$PSI_OMNI_SQLITE_LOCAL_CACHE dir=$PSI_OMNI_SQLITE_LOCAL_CACHE_DIR"
|
| 617 |
+
echo "OSS SQLite fallback: root=$PSI_OMNI_SQLITE_FALLBACK_ROOT lock_timeout=$PSI_OMNI_SQLITE_LOCAL_CACHE_LOCK_TIMEOUT"
|
| 618 |
+
echo "OSS full-shard cache: size=$PSI_OMNI_FULL_SHARD_CACHE_SIZE max_gib=$PSI_OMNI_FULL_SHARD_CACHE_MAX_GIB prefetch=$PSI_OMNI_FULL_SHARD_PREFETCH stagger=$PSI_OMNI_FULL_SHARD_PREFETCH_STAGGER log=$PSI_OMNI_FULL_SHARD_CACHE_LOG"
|
| 619 |
+
echo "OSS full-shard I/O slots: $PSI_OMNI_FULL_SHARD_IO_SLOTS dir=$PSI_OMNI_FULL_SHARD_IO_LOCK_DIR"
|
| 620 |
+
echo "OSS direct staging: enabled=$PSI_OMNI_DIRECT_OSS_STAGE parallel=$PSI_OMNI_OSSUTIL_PARALLEL dir=$PSI_OMNI_DIRECT_OSS_STAGE_DIR"
|
| 621 |
+
echo "Video motion local cache: enabled=$PSI_VIDEO_MOTION_LOCAL_CACHE dir=$PSI_VIDEO_MOTION_LOCAL_CACHE_DIR"
|
| 622 |
+
echo "Video motion next-shard prefetch: enabled=$PSI_VIDEO_MOTION_PREFETCH slots=$PSI_VIDEO_MOTION_PREFETCH_SLOTS dir=$PSI_VIDEO_MOTION_PREFETCH_LOCK_DIR"
|
| 623 |
+
echo "OSS shard reuse passes: $PSI_OMNI_OSS_SHARD_REUSE_PASSES"
|
| 624 |
+
echo "Foreach gradient-norm monitor: $PSI_FOREACH_GRAD_NORM"
|
| 625 |
+
echo "Omni hidden tensor cache size: $PSI_OMNI_HIDDEN_TENSOR_CACHE_SIZE"
|
| 626 |
+
echo "Omni safe-open reader cache size: $PSI_OMNI_SAFE_OPEN_CACHE_SIZE"
|
| 627 |
+
echo "Task weights: text=$PSI_TEXT_TASK_WEIGHT audio_text=$PSI_AUDIO_TEXT_TASK_WEIGHT audio_only=$PSI_AUDIO_ONLY_TASK_WEIGHT audio_command_text=$PSI_AUDIO_COMMAND_TEXT_TASK_WEIGHT audio_command_only=$PSI_AUDIO_COMMAND_ONLY_TASK_WEIGHT video_text=$PSI_VIDEO_TEXT_TASK_WEIGHT video_only=$PSI_VIDEO_ONLY_TASK_WEIGHT video_clip_text=$PSI_VIDEO_CLIP_TEXT_TASK_WEIGHT video_clip_only=$PSI_VIDEO_CLIP_ONLY_TASK_WEIGHT"
|
| 628 |
+
# echo "Resume run dir: $PSI_PRETRAIN_RUN_DIR"
|
| 629 |
+
echo "TextOp 39D action/state enabled"
|
| 630 |
+
echo "RTC training prefix: random 0-$RTC_PREFIX_FRAMES frames; deployment overlap: $RTC_PREFIX_FRAMES frames"
|
| 631 |
+
echo "Flow RNG: global-batch stratified sigma [$PSI_FLOW_SIGMA_MIN,$PSI_FLOW_SIGMA_MAX), rank-independent noise"
|
| 632 |
+
echo "Future progress embedding: enabled, dim $PROGRESS_DIM"
|
| 633 |
+
echo "Backbone path: $PSI_PRETRAIN_BACKBONE_PATH"
|
| 634 |
+
echo "Output dir: $PSI_OUTPUT_DIR"
|
| 635 |
+
echo "Aligned audio window seconds: $AUDIO_WINDOW_SECONDS"
|
| 636 |
+
echo "Aligned audio sample step frames: $AUDIO_SAMPLE_STEP_FRAMES"
|
| 637 |
+
echo "Omni hidden cache root: $PSI_OMNI_HIDDEN_CACHE_ROOT"
|
| 638 |
+
echo "Video clip trust metadata: $VIDEO_CLIP_TRUST_METADATA"
|
| 639 |
+
echo "Video clip records cache dir: $VIDEO_CLIP_RECORDS_CACHE_DIR"
|
| 640 |
+
echo "Video record build workers: $VIDEO_RECORD_BUILD_WORKERS"
|
| 641 |
+
echo "Video record build chunk size: $VIDEO_RECORD_BUILD_CHUNK_SIZE"
|
| 642 |
+
echo "Micro batch: train=$TRAIN_BATCH_SIZE val=$VAL_BATCH_SIZE grad_accum=$GRADIENT_ACCUMULATION_STEPS effective_train_batch=$EFFECTIVE_TRAIN_BATCH_SIZE"
|
| 643 |
+
args="posttrain_v1_seed_text_audio_video_mix_psi0_config \
|
| 644 |
+
--seed=$TRAIN_SEED \
|
| 645 |
+
--exp=posttrain \
|
| 646 |
+
--timestamp=$(date +"%y%m%d%H%M") \
|
| 647 |
+
--train.name=posttrain_v1_trimodal \
|
| 648 |
+
--train.data_parallel=ddp \
|
| 649 |
+
--train.mixed_precision=bf16 \
|
| 650 |
+
--train.train_batch_size=$TRAIN_BATCH_SIZE \
|
| 651 |
+
--train.val_batch_size=$VAL_BATCH_SIZE \
|
| 652 |
+
--train.num-workers=$DATALOADER_NUM_WORKERS \
|
| 653 |
+
--train.val-num-workers=$VAL_DATALOADER_NUM_WORKERS \
|
| 654 |
+
--train.resume_from_checkpoint=None \
|
| 655 |
+
--train.max_checkpoints_to_keep=1 \
|
| 656 |
+
--train.gradient_accumulation_steps=$GRADIENT_ACCUMULATION_STEPS \
|
| 657 |
+
--train.learning_rate=1e-4 \
|
| 658 |
+
--train.max_training_steps=$MAX_TRAINING_STEPS \
|
| 659 |
+
--train.warmup_ratio=None \
|
| 660 |
+
--train.warmup_steps=$WARMUP_STEPS \
|
| 661 |
+
--train.checkpointing_steps=$CHECKPOINTING_STEPS \
|
| 662 |
+
--train.validation_steps=$VALIDATION_STEPS \
|
| 663 |
+
--train.val_num_batches=$VAL_NUM_BATCHES \
|
| 664 |
+
--train.lr_scheduler_type=cosine_with_min_lr \
|
| 665 |
+
--train.lr_scheduler_kwargs.betas 0.9 0.999 \
|
| 666 |
+
--train.lr_scheduler_kwargs.weight_decay=0 \
|
| 667 |
+
--train.lr_scheduler_kwargs.eps=1e-8 \
|
| 668 |
+
--log.report_to=$PSI_LOG_REPORT_TO \
|
| 669 |
+
--train.output_dir=$PSI_OUTPUT_DIR \
|
| 670 |
+
--data.text.root-dir=$PSI_SEED_ROOT \
|
| 671 |
+
--data.text.metadata-csv=$PSI_SEED_METADATA \
|
| 672 |
+
--data.text.records-cache-dir=$PSI_TEXT_RECORDS_CACHE_DIR \
|
| 673 |
+
--data.text.split-source-namespace=seed \
|
| 674 |
+
--data.text.motion-cache-size=$TEXT_MOTION_CACHE_SIZE \
|
| 675 |
+
--data.text.use-omni-hidden-cache \
|
| 676 |
+
--data.text.require-omni-hidden-cache \
|
| 677 |
+
--data.text.omni-hidden-root=$PSI_OMNI_HIDDEN_CACHE_ROOT \
|
| 678 |
+
--data.text-ratio=$PSI_TEXT_TASK_WEIGHT \
|
| 679 |
+
--data.audio.sources-json=$PSI_AUDIO_SOURCES_JSON \
|
| 680 |
+
--data.audio-ratio=$PSI_AUDIO_TEXT_TASK_WEIGHT \
|
| 681 |
+
--data.audio-only-ratio=$PSI_AUDIO_ONLY_TASK_WEIGHT \
|
| 682 |
+
--data.audio.sample-stride=1 \
|
| 683 |
+
--data.audio.sample-step-frames=$AUDIO_SAMPLE_STEP_FRAMES \
|
| 684 |
+
--data.audio.motion-fps=$MOTION_FPS \
|
| 685 |
+
--data.audio.audio-sampling-rate=16000 \
|
| 686 |
+
--data.audio.audio-window-seconds=$AUDIO_WINDOW_SECONDS \
|
| 687 |
+
--data.audio.audio-left-context-seconds=0.0 \
|
| 688 |
+
--data.audio.use-omni-hidden-cache \
|
| 689 |
+
--data.audio.require-omni-hidden-cache \
|
| 690 |
+
--data.audio.omni-hidden-root=$PSI_OMNI_HIDDEN_CACHE_ROOT \
|
| 691 |
+
--data.audio-command.sources-json=$PSI_AUDIO_COMMAND_SOURCES_JSON \
|
| 692 |
+
--data.audio-command-ratio=$PSI_AUDIO_COMMAND_TEXT_TASK_WEIGHT \
|
| 693 |
+
--data.audio-command-only-ratio=$PSI_AUDIO_COMMAND_ONLY_TASK_WEIGHT \
|
| 694 |
+
--data.audio-command.sample-stride=1 \
|
| 695 |
+
--data.audio-command.sample-step-frames=$AUDIO_SAMPLE_STEP_FRAMES \
|
| 696 |
+
--data.audio-command.motion-fps=$MOTION_FPS \
|
| 697 |
+
--data.audio-command.audio-sampling-rate=16000 \
|
| 698 |
+
--data.audio-command.audio-window-seconds=$AUDIO_WINDOW_SECONDS \
|
| 699 |
+
--data.audio-command.audio-left-context-seconds=0.0 \
|
| 700 |
+
--data.audio-command.full-audio \
|
| 701 |
+
--data.audio-command.use-omni-hidden-cache \
|
| 702 |
+
--data.audio-command.require-omni-hidden-cache \
|
| 703 |
+
--data.audio-command.omni-hidden-root=$PSI_OMNI_HIDDEN_CACHE_ROOT \
|
| 704 |
+
--data.video-ratio=$PSI_VIDEO_TEXT_TASK_WEIGHT \
|
| 705 |
+
--data.video-only-ratio=$PSI_VIDEO_ONLY_TASK_WEIGHT \
|
| 706 |
+
--data.video.sample-stride=1 \
|
| 707 |
+
--data.video.motion-fps=$MOTION_FPS \
|
| 708 |
+
--data.video.video-sample-fps=$VIDEO_SAMPLE_FPS \
|
| 709 |
+
--data.video.video-size-pixels=$VIDEO_SIZE_PIXELS \
|
| 710 |
+
--data.video.video-decoder=$VIDEO_DECODER \
|
| 711 |
+
--data.video.video-cache-size=$VIDEO_CACHE_SIZE \
|
| 712 |
+
--data.video.record-build-workers=$VIDEO_RECORD_BUILD_WORKERS \
|
| 713 |
+
--data.video.record-build-chunk-size=$VIDEO_RECORD_BUILD_CHUNK_SIZE \
|
| 714 |
+
--data.video.sources-json=$PSI_VIDEO_SOURCES_JSON \
|
| 715 |
+
--data.video-clip-text-ratio=$PSI_VIDEO_CLIP_TEXT_TASK_WEIGHT \
|
| 716 |
+
--data.video-clip-only-ratio=$PSI_VIDEO_CLIP_ONLY_TASK_WEIGHT \
|
| 717 |
+
--data.video-clip.sources-json=$PSI_VIDEO_CLIP_SOURCES_JSON \
|
| 718 |
+
--data.video-clip.sample-stride=1 \
|
| 719 |
+
--data.video-clip.motion-fps=$MOTION_FPS \
|
| 720 |
+
--data.video-clip.video-sample-fps=$VIDEO_CLIP_SAMPLE_FPS \
|
| 721 |
+
--data.video-clip.video-size-pixels=$VIDEO_SIZE_PIXELS \
|
| 722 |
+
--data.video-clip.video-decoder=$VIDEO_DECODER \
|
| 723 |
+
--data.video-clip.video-cache-size=$VIDEO_CACHE_SIZE \
|
| 724 |
+
--data.video-clip.motion-cache-size=$VIDEO_CLIP_MOTION_CACHE_SIZE \
|
| 725 |
+
--data.video-clip.record-build-workers=$VIDEO_RECORD_BUILD_WORKERS \
|
| 726 |
+
--data.video-clip.record-build-chunk-size=$VIDEO_RECORD_BUILD_CHUNK_SIZE \
|
| 727 |
+
$VIDEO_CLIP_TRUST_METADATA_FLAG \
|
| 728 |
+
--data.video-clip.records-cache-dir=$VIDEO_CLIP_RECORDS_CACHE_DIR \
|
| 729 |
+
--data.video-clip.use-omni-hidden-cache \
|
| 730 |
+
--data.video-clip.require-omni-hidden-cache \
|
| 731 |
+
--data.video-clip.omni-hidden-root=$PSI_OMNI_HIDDEN_CACHE_ROOT \
|
| 732 |
+
--data.video-clip.action-exec-horizon=$ACTION_EXEC_HORIZON \
|
| 733 |
+
--data.sampler=task_mixture \
|
| 734 |
+
--data.val-task=text,audio,audio_command,video_clip \
|
| 735 |
+
--data.transform.repack.num-past-frames=$NUM_PAST_FRAMES \
|
| 736 |
+
--data.transform.repack.textop-actions \
|
| 737 |
+
--data.transform.repack.action-chunk-size=$ACTION_CHUNK_SIZE \
|
| 738 |
+
--data.transform.repack.rtc-prefix-frames=$RTC_PREFIX_FRAMES \
|
| 739 |
+
--data.transform.repack.pad-action-dim=39 \
|
| 740 |
+
--data.transform.repack.pad-state-dim=39 \
|
| 741 |
+
--data.transform.field.action_norm_type=bounds_q99 \
|
| 742 |
+
--data.transform.field.stat-path=assets/stats/seed_all_g1_motion39_textop_stats.json \
|
| 743 |
+
--data.transform.field.normalize-state \
|
| 744 |
+
--data.transform.field.pad-action-dim=39 \
|
| 745 |
+
--data.transform.field.pad-state-dim=39 \
|
| 746 |
+
--data.transform.model.resize.size 240 320 \
|
| 747 |
+
--data.transform.model.center_crop.size 240 320 \
|
| 748 |
+
--data.transform.model.no-img-aug \
|
| 749 |
+
--data.transform.model.text-only \
|
| 750 |
+
--data.transform.model.audio-only \
|
| 751 |
+
--data.transform.model.video-only \
|
| 752 |
+
--data.transform.model.video-size-pixels=$VIDEO_SIZE_PIXELS \
|
| 753 |
+
--model.model_name_or_path=$PSI_PRETRAIN_BACKBONE_PATH \
|
| 754 |
+
--model.noise-scheduler=flow \
|
| 755 |
+
--model.n_conditions=0 \
|
| 756 |
+
--model.action-chunk-size=$ACTION_CHUNK_SIZE \
|
| 757 |
+
--model.action-dim=39 \
|
| 758 |
+
--model.action-exec-horizon=$ACTION_EXEC_HORIZON \
|
| 759 |
+
--model.observation-horizon=$STATE_HORIZON \
|
| 760 |
+
--model.odim=39 \
|
| 761 |
+
--model.view_feature_dim=2048 \
|
| 762 |
+
--model.use-future-progress-embedding \
|
| 763 |
+
--model.future-progress-dim=$PROGRESS_DIM \
|
| 764 |
+
--model.no-use_film \
|
| 765 |
+
--model.no-combined_temb \
|
| 766 |
+
--model.rtc \
|
| 767 |
+
--model.max-delay=11 \
|
| 768 |
+
--train.loss-group-weight.joint-abs=1.0 \
|
| 769 |
+
--train.loss-group-weight.root-delta-xy=3.0 \
|
| 770 |
+
--train.loss-group-weight.root-rot5-delta=10.0 \
|
| 771 |
+
--train.loss-group-weight.root-height=15.0 \
|
| 772 |
+
--train.loss-group-weight.foot-contact=0.1
|
| 773 |
+
"
|
| 774 |
+
torchrun --nproc_per_node=$NPROC_PER_NODE --master_port=$MASTER_PORT scripts/train.py \
|
| 775 |
+
$args "$@"
|
posttrain-v1-psi0-all-7task-cached.sh
ADDED
|
@@ -0,0 +1,459 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
pkill -f monitor.py
|
| 2 |
+
|
| 3 |
+
PROJECT_ROOT=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
|
| 4 |
+
cd "$PROJECT_ROOT"
|
| 5 |
+
export PYTHONPATH="$PROJECT_ROOT/src${PYTHONPATH:+:$PYTHONPATH}"
|
| 6 |
+
export CUDA_VISIBLE_DEVICES=${CUDA_VISIBLE_DEVICES:-0,1,2,3,4,5,6,7}
|
| 7 |
+
export PSI_OUTPUT_DIR=${PSI_OUTPUT_DIR:-$PROJECT_ROOT/runs}
|
| 8 |
+
export PSI_LOG_REPORT_TO=${PSI_LOG_REPORT_TO:-wandb}
|
| 9 |
+
|
| 10 |
+
source "$PROJECT_ROOT/scripts/activate_psi_env.sh"
|
| 11 |
+
[ -f .env ] && source .env
|
| 12 |
+
|
| 13 |
+
|
| 14 |
+
export OMP_NUM_THREADS=8
|
| 15 |
+
export TF_CPP_MIN_LOG_LEVEL=3
|
| 16 |
+
export PYTORCH_CUDA_ALLOC_CONF=${PYTORCH_CUDA_ALLOC_CONF:-expandable_segments:True}
|
| 17 |
+
export PSI_SKIP_VLM_MODEL=${PSI_SKIP_VLM_MODEL:-1}
|
| 18 |
+
export PSI_TASK_MIXTURE_SYNC_RANKS=${PSI_TASK_MIXTURE_SYNC_RANKS:-0}
|
| 19 |
+
export PSI_TASK_MIXTURE_SHARD_AWARE=${PSI_TASK_MIXTURE_SHARD_AWARE:-0}
|
| 20 |
+
export PSI_OMNI_HIDDEN_TENSOR_CACHE_SIZE=${PSI_OMNI_HIDDEN_TENSOR_CACHE_SIZE:-64}
|
| 21 |
+
|
| 22 |
+
# Keep JSON sources readable in this script, but pass compact JSON to tyro.
|
| 23 |
+
compact_json() {
|
| 24 |
+
python -c 'import json, sys; print(json.dumps(json.load(sys.stdin), separators=(",", ":")))'
|
| 25 |
+
}
|
| 26 |
+
|
| 27 |
+
PSI_SEED_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/Psi0/dataset/seed_all
|
| 28 |
+
PSI_SEED_METADATA=metadata/seed_metadata_v004.csv
|
| 29 |
+
PSI_TEXT_RECORDS_CACHE_DIR=${PSI_TEXT_RECORDS_CACHE_DIR:-$PSI_SEED_ROOT/.index_cache_multimodal_yth}
|
| 30 |
+
|
| 31 |
+
PSI_DANCE_MOTION_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/GR00T-WholeBodyControl/gear_sonic_deploy/reference/converted/psi0_dance/gvhmr
|
| 32 |
+
PSI_DANCE_AUDIO_VIDEO_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/dance_1723_video_psi0/dance_1723_video_10s
|
| 33 |
+
PSI_DANCE_VIDEO_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/dance_1723_video_psi0/dance_1723_video_10s
|
| 34 |
+
PSI_DANCE_VIDEO_EMBED_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/dance_1723_video_psi0/dance_1723_video_10s_embeds
|
| 35 |
+
PSI_VIDEO_DEMO_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/videogen/motion_example_0506_demo
|
| 36 |
+
PSI_2264_WITH_AUDIO_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/videogen/2264_with_audio
|
| 37 |
+
PSI_QWEN3_TTS_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/qwen3-tts
|
| 38 |
+
PSI_VIDEO_CLIP_DATASET_ROOT=${PSI_VIDEO_CLIP_DATASET_ROOT:-/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/videogen/psi0_video_clip_50f_dense_stride1_direct_full}
|
| 39 |
+
PSI_DANCE_VIDEO_CLIP_ROOT=$PSI_VIDEO_CLIP_DATASET_ROOT/dance_1723_video_10s
|
| 40 |
+
PSI_MOTION_EXAMPLE_VIDEO_CLIP_ROOT=$PSI_VIDEO_CLIP_DATASET_ROOT/motion_example_0506_demo
|
| 41 |
+
PSI_2264_WITH_AUDIO_VIDEO_CLIP_ROOT=$PSI_VIDEO_CLIP_DATASET_ROOT/2264_with_audio
|
| 42 |
+
PSI_DANCE_VIDEO_CLIP_EMBED_ROOT=${PSI_DANCE_VIDEO_CLIP_EMBED_ROOT:-$PSI_DANCE_VIDEO_CLIP_ROOT/embeds}
|
| 43 |
+
PSI_MOTION_EXAMPLE_VIDEO_CLIP_EMBED_ROOT=${PSI_MOTION_EXAMPLE_VIDEO_CLIP_EMBED_ROOT:-$PSI_MOTION_EXAMPLE_VIDEO_CLIP_ROOT/embeds}
|
| 44 |
+
PSI_2264_WITH_AUDIO_VIDEO_CLIP_EMBED_ROOT=${PSI_2264_WITH_AUDIO_VIDEO_CLIP_EMBED_ROOT:-$PSI_2264_WITH_AUDIO_VIDEO_CLIP_ROOT/embeds}
|
| 45 |
+
PSI_MOTION_EXAMPLE_VIDEO_EMBED_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/videogen/motion_example_0506_demo_video_embeds
|
| 46 |
+
PSI_AUDIO_RECORDS_CACHE_DIR=${PSI_AUDIO_RECORDS_CACHE_DIR:-$PSI_2264_WITH_AUDIO_ROOT/.index_cache_multimodal_yth}
|
| 47 |
+
PSI_AUDIO_COMMAND_RECORDS_CACHE_DIR=${PSI_AUDIO_COMMAND_RECORDS_CACHE_DIR:-$PSI_QWEN3_TTS_ROOT/.index_cache_multimodal_yth}
|
| 48 |
+
PSI_DANCE_AUDIO_NPY_ROOT=${PSI_DANCE_AUDIO_NPY_ROOT:-$PSI_DANCE_AUDIO_VIDEO_ROOT/audio_npy_yth}
|
| 49 |
+
PSI_DANCE_AUDIO_NPY_MANIFEST=${PSI_DANCE_AUDIO_NPY_MANIFEST:-$PSI_DANCE_AUDIO_VIDEO_ROOT/metadata_audio_npy.jsonl}
|
| 50 |
+
PSI_2264_AUDIO_NPY_ROOT=${PSI_2264_AUDIO_NPY_ROOT:-$PSI_2264_WITH_AUDIO_ROOT/audio_npy_yth}
|
| 51 |
+
PSI_2264_AUDIO_NPY_MANIFEST=${PSI_2264_AUDIO_NPY_MANIFEST:-$PSI_2264_WITH_AUDIO_ROOT/metadata_audio_npy.jsonl}
|
| 52 |
+
PSI_QWEN3_TTS_AUDIO_NPY_ROOT=${PSI_QWEN3_TTS_AUDIO_NPY_ROOT:-$PSI_QWEN3_TTS_ROOT/audio_npy_yth}
|
| 53 |
+
PSI_QWEN3_TTS_AUDIO_NPY_MANIFEST=${PSI_QWEN3_TTS_AUDIO_NPY_MANIFEST:-$PSI_QWEN3_TTS_ROOT/metadata_audio_npy.jsonl}
|
| 54 |
+
PSI_OMNI_HIDDEN_CACHE_ROOT=${PSI_OMNI_HIDDEN_CACHE_ROOT:-/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/videogen/omni_hidden_states_v23_oss}
|
| 55 |
+
|
| 56 |
+
if [ -f "$PSI_2264_AUDIO_NPY_MANIFEST" ]; then
|
| 57 |
+
PSI_2264_AUDIO_METADATA_JSONL=$PSI_2264_AUDIO_NPY_MANIFEST
|
| 58 |
+
PSI_2264_AUDIO_PATH_KEY=audio_path
|
| 59 |
+
else
|
| 60 |
+
PSI_2264_AUDIO_METADATA_JSONL=metadata_convert.jsonl
|
| 61 |
+
PSI_2264_AUDIO_PATH_KEY=video_path
|
| 62 |
+
fi
|
| 63 |
+
|
| 64 |
+
if [ -f "$PSI_DANCE_AUDIO_NPY_MANIFEST" ]; then
|
| 65 |
+
PSI_DANCE_AUDIO_SOURCE_JSON=$(cat <<JSON
|
| 66 |
+
{
|
| 67 |
+
"name": "psi0_dance_audio",
|
| 68 |
+
"dataset_root": "$PSI_DANCE_AUDIO_VIDEO_ROOT",
|
| 69 |
+
"metadata_jsonl": "$PSI_DANCE_AUDIO_NPY_MANIFEST",
|
| 70 |
+
"motion_dir_key": "converted_path",
|
| 71 |
+
"audio_path_key": "audio_path",
|
| 72 |
+
"records_cache_dir": "$PSI_AUDIO_RECORDS_CACHE_DIR",
|
| 73 |
+
"dataset_name": "seed-audio-g1"
|
| 74 |
+
}
|
| 75 |
+
JSON
|
| 76 |
+
)
|
| 77 |
+
else
|
| 78 |
+
PSI_DANCE_AUDIO_SOURCE_JSON=$(cat <<JSON
|
| 79 |
+
{
|
| 80 |
+
"name": "psi0_dance_audio",
|
| 81 |
+
"root_dir": "$PSI_DANCE_MOTION_ROOT",
|
| 82 |
+
"video_dir": "$PSI_DANCE_AUDIO_VIDEO_ROOT",
|
| 83 |
+
"records_cache_dir": "$PSI_AUDIO_RECORDS_CACHE_DIR",
|
| 84 |
+
"dataset_name": "seed-audio-g1"
|
| 85 |
+
}
|
| 86 |
+
JSON
|
| 87 |
+
)
|
| 88 |
+
fi
|
| 89 |
+
|
| 90 |
+
if [ -z "$PSI_AUDIO_SOURCES_JSON" ]; then
|
| 91 |
+
PSI_AUDIO_SOURCES_JSON=$(compact_json <<JSON
|
| 92 |
+
[
|
| 93 |
+
$PSI_DANCE_AUDIO_SOURCE_JSON,
|
| 94 |
+
{
|
| 95 |
+
"name": "2264_with_audio",
|
| 96 |
+
"dataset_root": "$PSI_2264_WITH_AUDIO_ROOT",
|
| 97 |
+
"metadata_jsonl": "$PSI_2264_AUDIO_METADATA_JSONL",
|
| 98 |
+
"motion_dir_key": "converted_path",
|
| 99 |
+
"audio_path_key": "$PSI_2264_AUDIO_PATH_KEY",
|
| 100 |
+
"records_cache_dir": "$PSI_AUDIO_RECORDS_CACHE_DIR",
|
| 101 |
+
"dataset_name": "2264-with-audio-g1"
|
| 102 |
+
}
|
| 103 |
+
]
|
| 104 |
+
JSON
|
| 105 |
+
)
|
| 106 |
+
fi
|
| 107 |
+
|
| 108 |
+
if [ -f "$PSI_QWEN3_TTS_AUDIO_NPY_MANIFEST" ]; then
|
| 109 |
+
PSI_AUDIO_COMMAND_SOURCE_JSON=$(cat <<JSON
|
| 110 |
+
{
|
| 111 |
+
"name": "qwen3_tts_seed_audio_command",
|
| 112 |
+
"root_dir": "$PSI_SEED_ROOT",
|
| 113 |
+
"dataset_root": "$PSI_QWEN3_TTS_ROOT",
|
| 114 |
+
"metadata_jsonl": "$PSI_QWEN3_TTS_AUDIO_NPY_MANIFEST",
|
| 115 |
+
"motion_dir_key": "filename",
|
| 116 |
+
"audio_path_key": "audio_path",
|
| 117 |
+
"instruction_key": "content_short_description",
|
| 118 |
+
"motion_metadata_csv": "$PSI_SEED_METADATA",
|
| 119 |
+
"motion_metadata_key": "filename",
|
| 120 |
+
"motion_metadata_path_key": "move_g1_path",
|
| 121 |
+
"audio_file_extensions": ".npy",
|
| 122 |
+
"records_cache_dir": "$PSI_AUDIO_COMMAND_RECORDS_CACHE_DIR",
|
| 123 |
+
"audio_task": "command",
|
| 124 |
+
"dataset_name": "seed-audio-command-g1"
|
| 125 |
+
}
|
| 126 |
+
JSON
|
| 127 |
+
)
|
| 128 |
+
else
|
| 129 |
+
PSI_AUDIO_COMMAND_SOURCE_JSON=$(cat <<JSON
|
| 130 |
+
{
|
| 131 |
+
"name": "qwen3_tts_seed_audio_command",
|
| 132 |
+
"root_dir": "$PSI_SEED_ROOT",
|
| 133 |
+
"dataset_root": "$PSI_QWEN3_TTS_ROOT",
|
| 134 |
+
"metadata_jsonl": "metadata.jsonl",
|
| 135 |
+
"motion_dir_key": "filename",
|
| 136 |
+
"audio_path_key": "audio_path",
|
| 137 |
+
"instruction_key": "content_short_description",
|
| 138 |
+
"motion_metadata_csv": "$PSI_SEED_METADATA",
|
| 139 |
+
"motion_metadata_key": "filename",
|
| 140 |
+
"motion_metadata_path_key": "move_g1_path",
|
| 141 |
+
"audio_file_extensions": ".wav",
|
| 142 |
+
"records_cache_dir": "$PSI_AUDIO_COMMAND_RECORDS_CACHE_DIR",
|
| 143 |
+
"audio_task": "command",
|
| 144 |
+
"dataset_name": "seed-audio-command-g1"
|
| 145 |
+
}
|
| 146 |
+
JSON
|
| 147 |
+
)
|
| 148 |
+
fi
|
| 149 |
+
|
| 150 |
+
if [ -z "$PSI_AUDIO_COMMAND_SOURCES_JSON" ]; then
|
| 151 |
+
PSI_AUDIO_COMMAND_SOURCES_JSON=$(compact_json <<JSON
|
| 152 |
+
[
|
| 153 |
+
$PSI_AUDIO_COMMAND_SOURCE_JSON
|
| 154 |
+
]
|
| 155 |
+
JSON
|
| 156 |
+
)
|
| 157 |
+
fi
|
| 158 |
+
|
| 159 |
+
if [ -z "$PSI_VIDEO_SOURCES_JSON" ]; then
|
| 160 |
+
PSI_VIDEO_SOURCES_JSON=$(compact_json <<JSON
|
| 161 |
+
[
|
| 162 |
+
{
|
| 163 |
+
"name": "psi0_dance_gvhmr",
|
| 164 |
+
"root_dir": "$PSI_DANCE_MOTION_ROOT",
|
| 165 |
+
"video_dir": "$PSI_DANCE_VIDEO_ROOT",
|
| 166 |
+
"video_embed_dir": "$PSI_DANCE_VIDEO_EMBED_ROOT",
|
| 167 |
+
"dataset_name": "seed-video-g1"
|
| 168 |
+
},
|
| 169 |
+
{
|
| 170 |
+
"name": "motion_example_0506_demo",
|
| 171 |
+
"dataset_root": "$PSI_VIDEO_DEMO_ROOT",
|
| 172 |
+
"metadata_jsonl": "metadata.jsonl",
|
| 173 |
+
"motion_dir_key": "converted_path",
|
| 174 |
+
"video_path_key": "video_path",
|
| 175 |
+
"instruction_key": "video_caption",
|
| 176 |
+
"video_embed_dir": "$PSI_MOTION_EXAMPLE_VIDEO_EMBED_ROOT",
|
| 177 |
+
"dataset_name": "motion-example-video-g1"
|
| 178 |
+
}
|
| 179 |
+
]
|
| 180 |
+
JSON
|
| 181 |
+
)
|
| 182 |
+
fi
|
| 183 |
+
|
| 184 |
+
if [ -z "$PSI_VIDEO_CLIP_SOURCES_JSON" ]; then
|
| 185 |
+
PSI_VIDEO_CLIP_SOURCES_JSON=$(compact_json <<JSON
|
| 186 |
+
[
|
| 187 |
+
{
|
| 188 |
+
"name": "dance_1723_video_10s_clip50",
|
| 189 |
+
"dataset_root": "$PSI_DANCE_VIDEO_CLIP_ROOT",
|
| 190 |
+
"metadata_jsonl": "metadata.jsonl",
|
| 191 |
+
"motion_dir_key": "converted_path",
|
| 192 |
+
"video_path_key": "video_path",
|
| 193 |
+
"instruction_key": "video_caption",
|
| 194 |
+
"video_embed_dir": "$PSI_DANCE_VIDEO_CLIP_EMBED_ROOT",
|
| 195 |
+
"dataset_name": "dance-video-clip-g1"
|
| 196 |
+
},
|
| 197 |
+
{
|
| 198 |
+
"name": "motion_example_0506_demo_clip50",
|
| 199 |
+
"dataset_root": "$PSI_MOTION_EXAMPLE_VIDEO_CLIP_ROOT",
|
| 200 |
+
"metadata_jsonl": "metadata.jsonl",
|
| 201 |
+
"motion_dir_key": "converted_path",
|
| 202 |
+
"video_path_key": "video_path",
|
| 203 |
+
"instruction_key": "video_caption",
|
| 204 |
+
"video_embed_dir": "$PSI_MOTION_EXAMPLE_VIDEO_CLIP_EMBED_ROOT",
|
| 205 |
+
"dataset_name": "motion-example-video-clip-g1"
|
| 206 |
+
},
|
| 207 |
+
{
|
| 208 |
+
"name": "2264_with_audio_clip50",
|
| 209 |
+
"dataset_root": "$PSI_2264_WITH_AUDIO_VIDEO_CLIP_ROOT",
|
| 210 |
+
"metadata_jsonl": "metadata.jsonl",
|
| 211 |
+
"motion_dir_key": "converted_path",
|
| 212 |
+
"video_path_key": "video_path",
|
| 213 |
+
"instruction_key": "video_caption",
|
| 214 |
+
"video_embed_dir": "$PSI_2264_WITH_AUDIO_VIDEO_CLIP_EMBED_ROOT",
|
| 215 |
+
"dataset_name": "2264-with-audio-video-clip-g1"
|
| 216 |
+
}
|
| 217 |
+
]
|
| 218 |
+
JSON
|
| 219 |
+
)
|
| 220 |
+
fi
|
| 221 |
+
|
| 222 |
+
# Prompted-cache mix, excluding legacy VIDEO_TEXT / VIDEO_ONLY and all *_only
|
| 223 |
+
# tasks. The *_only cache directories are intentionally not required.
|
| 224 |
+
# Smaller data sources get larger weights. These use a tempered inverse-size
|
| 225 |
+
# rule, (47M / source_size)^0.25, so rare modalities are upweighted without
|
| 226 |
+
# completely starving text/video_clip in BatchMixtureSampler.
|
| 227 |
+
# text stride1 frame samples: ~47,000,000
|
| 228 |
+
# music audio dance/audio frame samples use dense stride1 like video_clip
|
| 229 |
+
# audio_command wav files: 41,760
|
| 230 |
+
# dense video_clip metadata: 4,597,576
|
| 231 |
+
PSI_TEXT_TASK_WEIGHT=1
|
| 232 |
+
PSI_AUDIO_TEXT_TASK_WEIGHT=1
|
| 233 |
+
PSI_AUDIO_ONLY_TASK_WEIGHT=0
|
| 234 |
+
PSI_AUDIO_COMMAND_TEXT_TASK_WEIGHT=1
|
| 235 |
+
PSI_AUDIO_COMMAND_ONLY_TASK_WEIGHT=0
|
| 236 |
+
PSI_VIDEO_TEXT_TASK_WEIGHT=0
|
| 237 |
+
PSI_VIDEO_ONLY_TASK_WEIGHT=0
|
| 238 |
+
PSI_VIDEO_CLIP_TEXT_TASK_WEIGHT=1
|
| 239 |
+
PSI_VIDEO_CLIP_ONLY_TASK_WEIGHT=0
|
| 240 |
+
|
| 241 |
+
PSI_PRETRAIN_BACKBONE_PATH=/cpfs01/VideoGen/public_ckpt/Qwen/Qwen2.5-Omni-3B
|
| 242 |
+
NUM_PAST_FRAMES=0
|
| 243 |
+
STATE_HORIZON=$((NUM_PAST_FRAMES + 1))
|
| 244 |
+
ACTION_CHUNK_SIZE=30
|
| 245 |
+
ACTION_EXEC_HORIZON=20
|
| 246 |
+
RTC_PREFIX_FRAMES=$((ACTION_CHUNK_SIZE - ACTION_EXEC_HORIZON))
|
| 247 |
+
PROGRESS_DIM=5
|
| 248 |
+
MOTION_FPS=50
|
| 249 |
+
AUDIO_WINDOW_SECONDS=$(awk "BEGIN {printf \"%.6f\", ${ACTION_CHUNK_SIZE} / ${MOTION_FPS}}")
|
| 250 |
+
AUDIO_SAMPLE_STEP_FRAMES=1
|
| 251 |
+
VIDEO_SAMPLE_FPS=1.0
|
| 252 |
+
VIDEO_CLIP_SAMPLE_FPS=8.0
|
| 253 |
+
VIDEO_SIZE_PIXELS=50176
|
| 254 |
+
VIDEO_DECODER=ffmpeg
|
| 255 |
+
VIDEO_CACHE_SIZE=64
|
| 256 |
+
VIDEO_CLIP_MOTION_CACHE_SIZE=256
|
| 257 |
+
VIDEO_RECORD_BUILD_WORKERS=${VIDEO_RECORD_BUILD_WORKERS:-32}
|
| 258 |
+
VIDEO_RECORD_BUILD_CHUNK_SIZE=${VIDEO_RECORD_BUILD_CHUNK_SIZE:-4096}
|
| 259 |
+
VIDEO_CLIP_TRUST_METADATA=${VIDEO_CLIP_TRUST_METADATA:-1}
|
| 260 |
+
VIDEO_CLIP_RECORDS_CACHE_DIR=${VIDEO_CLIP_RECORDS_CACHE_DIR:-$PSI_VIDEO_CLIP_DATASET_ROOT/.index_cache_multimodal_yth}
|
| 261 |
+
if [ "$VIDEO_CLIP_TRUST_METADATA" = "1" ] || [ "$VIDEO_CLIP_TRUST_METADATA" = "true" ] || [ "$VIDEO_CLIP_TRUST_METADATA" = "True" ]; then
|
| 262 |
+
VIDEO_CLIP_TRUST_METADATA_FLAG="--data.video-clip.trust-metadata"
|
| 263 |
+
else
|
| 264 |
+
VIDEO_CLIP_TRUST_METADATA_FLAG="--data.video-clip.no-trust-metadata"
|
| 265 |
+
fi
|
| 266 |
+
|
| 267 |
+
# DATALOADER_NUM_WORKERS=4
|
| 268 |
+
# VAL_DATALOADER_NUM_WORKERS=2
|
| 269 |
+
|
| 270 |
+
# DATALOADER_NUM_WORKERS=${DATALOADER_NUM_WORKERS:-0}
|
| 271 |
+
# VAL_DATALOADER_NUM_WORKERS=${VAL_DATALOADER_NUM_WORKERS:-0}
|
| 272 |
+
|
| 273 |
+
DATALOADER_NUM_WORKERS=${DATALOADER_NUM_WORKERS:-2}
|
| 274 |
+
VAL_DATALOADER_NUM_WORKERS=${VAL_DATALOADER_NUM_WORKERS:-2}
|
| 275 |
+
TRAIN_BATCH_SIZE=${TRAIN_BATCH_SIZE:-128}
|
| 276 |
+
VAL_BATCH_SIZE=${VAL_BATCH_SIZE:-128}
|
| 277 |
+
GRADIENT_ACCUMULATION_STEPS=${GRADIENT_ACCUMULATION_STEPS:-1}
|
| 278 |
+
MAX_TRAINING_STEPS=${MAX_TRAINING_STEPS:-500000}
|
| 279 |
+
WARMUP_STEPS=${WARMUP_STEPS:-1000}
|
| 280 |
+
CHECKPOINTING_STEPS=${CHECKPOINTING_STEPS:-1000}
|
| 281 |
+
VALIDATION_STEPS=${VALIDATION_STEPS:-1000}
|
| 282 |
+
VAL_NUM_BATCHES=${VAL_NUM_BATCHES:-20}
|
| 283 |
+
MASTER_PORT=${MASTER_PORT:-29500}
|
| 284 |
+
|
| 285 |
+
NPROC_PER_NODE=$(echo $CUDA_VISIBLE_DEVICES | tr ',' '\n' | wc -l)
|
| 286 |
+
EFFECTIVE_TRAIN_BATCH_SIZE=$((TRAIN_BATCH_SIZE * NPROC_PER_NODE * GRADIENT_ACCUMULATION_STEPS))
|
| 287 |
+
|
| 288 |
+
echo "Training with $NPROC_PER_NODE GPU(s): $CUDA_VISIBLE_DEVICES"
|
| 289 |
+
echo "Text root: $PSI_SEED_ROOT"
|
| 290 |
+
echo "Text records cache dir: $PSI_TEXT_RECORDS_CACHE_DIR"
|
| 291 |
+
echo "Dance motion root: $PSI_DANCE_MOTION_ROOT"
|
| 292 |
+
echo "Dance audio/video root: $PSI_DANCE_AUDIO_VIDEO_ROOT"
|
| 293 |
+
echo "Dance video root: $PSI_DANCE_VIDEO_ROOT"
|
| 294 |
+
echo "Dance video embed root: $PSI_DANCE_VIDEO_EMBED_ROOT"
|
| 295 |
+
echo "2264 with audio root: $PSI_2264_WITH_AUDIO_ROOT"
|
| 296 |
+
echo "Dance audio npy root: $PSI_DANCE_AUDIO_NPY_ROOT"
|
| 297 |
+
echo "Dance audio manifest: $PSI_DANCE_AUDIO_NPY_MANIFEST"
|
| 298 |
+
echo "2264 audio npy root: $PSI_2264_AUDIO_NPY_ROOT"
|
| 299 |
+
echo "2264 audio manifest: $PSI_2264_AUDIO_METADATA_JSONL"
|
| 300 |
+
echo "Audio records cache dir: $PSI_AUDIO_RECORDS_CACHE_DIR"
|
| 301 |
+
echo "Audio command records cache dir: $PSI_AUDIO_COMMAND_RECORDS_CACHE_DIR"
|
| 302 |
+
echo "Qwen3-TTS acdudio command root: $PSI_QWEN3_TTS_ROOT"
|
| 303 |
+
echo "Qwen3-TTS audio npy root: $PSI_QWEN3_TTS_AUDIO_NPY_ROOT"
|
| 304 |
+
echo "Qwen3-TTS audio manifest: $PSI_QWEN3_TTS_AUDIO_NPY_MANIFEST"
|
| 305 |
+
echo "Audio sources: $PSI_AUDIO_SOURCES_JSON"
|
| 306 |
+
echo "Audio command sources: $PSI_AUDIO_COMMAND_SOURCES_JSON"
|
| 307 |
+
echo "Video sources: $PSI_VIDEO_SOURCES_JSON"
|
| 308 |
+
echo "Video clip sources: $PSI_VIDEO_CLIP_SOURCES_JSON"
|
| 309 |
+
echo "DataLoader workers: train=$DATALOADER_NUM_WORKERS val=$VAL_DATALOADER_NUM_WORKERS"
|
| 310 |
+
echo "Sampler: task_mixture"
|
| 311 |
+
echo "Task mixture sync ranks: $PSI_TASK_MIXTURE_SYNC_RANKS"
|
| 312 |
+
echo "Task mixture shard aware: $PSI_TASK_MIXTURE_SHARD_AWARE"
|
| 313 |
+
echo "Omni hidden tensor cache size: $PSI_OMNI_HIDDEN_TENSOR_CACHE_SIZE"
|
| 314 |
+
echo "Task weights: text=$PSI_TEXT_TASK_WEIGHT audio_text=$PSI_AUDIO_TEXT_TASK_WEIGHT audio_only=$PSI_AUDIO_ONLY_TASK_WEIGHT audio_command_text=$PSI_AUDIO_COMMAND_TEXT_TASK_WEIGHT audio_command_only=$PSI_AUDIO_COMMAND_ONLY_TASK_WEIGHT video_text=$PSI_VIDEO_TEXT_TASK_WEIGHT video_only=$PSI_VIDEO_ONLY_TASK_WEIGHT video_clip_text=$PSI_VIDEO_CLIP_TEXT_TASK_WEIGHT video_clip_only=$PSI_VIDEO_CLIP_ONLY_TASK_WEIGHT"
|
| 315 |
+
# echo "Resume run dir: $PSI_PRETRAIN_RUN_DIR"
|
| 316 |
+
echo "TextOp 39D action/state enabled"
|
| 317 |
+
echo "Future progress embedding: enabled, dim $PROGRESS_DIM"
|
| 318 |
+
echo "Backbone path: $PSI_PRETRAIN_BACKBONE_PATH"
|
| 319 |
+
echo "Output dir: $PSI_OUTPUT_DIR"
|
| 320 |
+
echo "Aligned audio window seconds: $AUDIO_WINDOW_SECONDS"
|
| 321 |
+
echo "Aligned audio sample step frames: $AUDIO_SAMPLE_STEP_FRAMES"
|
| 322 |
+
echo "Omni hidden cache root: $PSI_OMNI_HIDDEN_CACHE_ROOT"
|
| 323 |
+
echo "Video clip trust metadata: $VIDEO_CLIP_TRUST_METADATA"
|
| 324 |
+
echo "Video clip records cache dir: $VIDEO_CLIP_RECORDS_CACHE_DIR"
|
| 325 |
+
echo "Video record build workers: $VIDEO_RECORD_BUILD_WORKERS"
|
| 326 |
+
echo "Video record build chunk size: $VIDEO_RECORD_BUILD_CHUNK_SIZE"
|
| 327 |
+
echo "Micro batch: train=$TRAIN_BATCH_SIZE val=$VAL_BATCH_SIZE grad_accum=$GRADIENT_ACCUMULATION_STEPS effective_train_batch=$EFFECTIVE_TRAIN_BATCH_SIZE"
|
| 328 |
+
|
| 329 |
+
args="posttrain_v1_seed_text_audio_video_mix_psi0_config \
|
| 330 |
+
--seed=292285 \
|
| 331 |
+
--exp=posttrain \
|
| 332 |
+
--timestamp=$(date +"%y%m%d%H%M") \
|
| 333 |
+
--train.name=posttrain_v1_trimodal \
|
| 334 |
+
--train.data_parallel=ddp \
|
| 335 |
+
--train.mixed_precision=bf16 \
|
| 336 |
+
--train.train_batch_size=$TRAIN_BATCH_SIZE \
|
| 337 |
+
--train.val_batch_size=$VAL_BATCH_SIZE \
|
| 338 |
+
--train.num-workers=$DATALOADER_NUM_WORKERS \
|
| 339 |
+
--train.val-num-workers=$VAL_DATALOADER_NUM_WORKERS \
|
| 340 |
+
--train.resume_from_checkpoint=None \
|
| 341 |
+
--train.max_checkpoints_to_keep=1 \
|
| 342 |
+
--train.gradient_accumulation_steps=$GRADIENT_ACCUMULATION_STEPS \
|
| 343 |
+
--train.learning_rate=1e-4 \
|
| 344 |
+
--train.max_training_steps=$MAX_TRAINING_STEPS \
|
| 345 |
+
--train.warmup_ratio=None \
|
| 346 |
+
--train.warmup_steps=$WARMUP_STEPS \
|
| 347 |
+
--train.checkpointing_steps=$CHECKPOINTING_STEPS \
|
| 348 |
+
--train.validation_steps=$VALIDATION_STEPS \
|
| 349 |
+
--train.val_num_batches=$VAL_NUM_BATCHES \
|
| 350 |
+
--train.lr_scheduler_type=cosine_with_min_lr \
|
| 351 |
+
--train.lr_scheduler_kwargs.betas 0.9 0.999 \
|
| 352 |
+
--train.lr_scheduler_kwargs.weight_decay=0 \
|
| 353 |
+
--train.lr_scheduler_kwargs.eps=1e-8 \
|
| 354 |
+
--log.report_to=$PSI_LOG_REPORT_TO \
|
| 355 |
+
--train.output_dir=$PSI_OUTPUT_DIR \
|
| 356 |
+
--data.text.root-dir=$PSI_SEED_ROOT \
|
| 357 |
+
--data.text.metadata-csv=$PSI_SEED_METADATA \
|
| 358 |
+
--data.text.records-cache-dir=$PSI_TEXT_RECORDS_CACHE_DIR \
|
| 359 |
+
--data.text.use-omni-hidden-cache \
|
| 360 |
+
--data.text.require-omni-hidden-cache \
|
| 361 |
+
--data.text.omni-hidden-root=$PSI_OMNI_HIDDEN_CACHE_ROOT \
|
| 362 |
+
--data.text-ratio=$PSI_TEXT_TASK_WEIGHT \
|
| 363 |
+
--data.audio.sources-json=$PSI_AUDIO_SOURCES_JSON \
|
| 364 |
+
--data.audio-ratio=$PSI_AUDIO_TEXT_TASK_WEIGHT \
|
| 365 |
+
--data.audio-only-ratio=$PSI_AUDIO_ONLY_TASK_WEIGHT \
|
| 366 |
+
--data.audio.sample-stride=1 \
|
| 367 |
+
--data.audio.sample-step-frames=$AUDIO_SAMPLE_STEP_FRAMES \
|
| 368 |
+
--data.audio.motion-fps=$MOTION_FPS \
|
| 369 |
+
--data.audio.audio-sampling-rate=16000 \
|
| 370 |
+
--data.audio.audio-window-seconds=$AUDIO_WINDOW_SECONDS \
|
| 371 |
+
--data.audio.audio-left-context-seconds=0.0 \
|
| 372 |
+
--data.audio.use-omni-hidden-cache \
|
| 373 |
+
--data.audio.require-omni-hidden-cache \
|
| 374 |
+
--data.audio.omni-hidden-root=$PSI_OMNI_HIDDEN_CACHE_ROOT \
|
| 375 |
+
--data.audio-command.sources-json=$PSI_AUDIO_COMMAND_SOURCES_JSON \
|
| 376 |
+
--data.audio-command-ratio=$PSI_AUDIO_COMMAND_TEXT_TASK_WEIGHT \
|
| 377 |
+
--data.audio-command-only-ratio=$PSI_AUDIO_COMMAND_ONLY_TASK_WEIGHT \
|
| 378 |
+
--data.audio-command.sample-stride=1 \
|
| 379 |
+
--data.audio-command.sample-step-frames=$AUDIO_SAMPLE_STEP_FRAMES \
|
| 380 |
+
--data.audio-command.motion-fps=$MOTION_FPS \
|
| 381 |
+
--data.audio-command.audio-sampling-rate=16000 \
|
| 382 |
+
--data.audio-command.audio-window-seconds=$AUDIO_WINDOW_SECONDS \
|
| 383 |
+
--data.audio-command.audio-left-context-seconds=0.0 \
|
| 384 |
+
--data.audio-command.full-audio \
|
| 385 |
+
--data.audio-command.use-omni-hidden-cache \
|
| 386 |
+
--data.audio-command.require-omni-hidden-cache \
|
| 387 |
+
--data.audio-command.omni-hidden-root=$PSI_OMNI_HIDDEN_CACHE_ROOT \
|
| 388 |
+
--data.video-ratio=$PSI_VIDEO_TEXT_TASK_WEIGHT \
|
| 389 |
+
--data.video-only-ratio=$PSI_VIDEO_ONLY_TASK_WEIGHT \
|
| 390 |
+
--data.video.sample-stride=1 \
|
| 391 |
+
--data.video.motion-fps=$MOTION_FPS \
|
| 392 |
+
--data.video.video-sample-fps=$VIDEO_SAMPLE_FPS \
|
| 393 |
+
--data.video.video-size-pixels=$VIDEO_SIZE_PIXELS \
|
| 394 |
+
--data.video.video-decoder=$VIDEO_DECODER \
|
| 395 |
+
--data.video.video-cache-size=$VIDEO_CACHE_SIZE \
|
| 396 |
+
--data.video.record-build-workers=$VIDEO_RECORD_BUILD_WORKERS \
|
| 397 |
+
--data.video.record-build-chunk-size=$VIDEO_RECORD_BUILD_CHUNK_SIZE \
|
| 398 |
+
--data.video.sources-json=$PSI_VIDEO_SOURCES_JSON \
|
| 399 |
+
--data.video-clip-text-ratio=$PSI_VIDEO_CLIP_TEXT_TASK_WEIGHT \
|
| 400 |
+
--data.video-clip-only-ratio=$PSI_VIDEO_CLIP_ONLY_TASK_WEIGHT \
|
| 401 |
+
--data.video-clip.sources-json=$PSI_VIDEO_CLIP_SOURCES_JSON \
|
| 402 |
+
--data.video-clip.sample-stride=1 \
|
| 403 |
+
--data.video-clip.motion-fps=$MOTION_FPS \
|
| 404 |
+
--data.video-clip.video-sample-fps=$VIDEO_CLIP_SAMPLE_FPS \
|
| 405 |
+
--data.video-clip.video-size-pixels=$VIDEO_SIZE_PIXELS \
|
| 406 |
+
--data.video-clip.video-decoder=$VIDEO_DECODER \
|
| 407 |
+
--data.video-clip.video-cache-size=$VIDEO_CACHE_SIZE \
|
| 408 |
+
--data.video-clip.motion-cache-size=$VIDEO_CLIP_MOTION_CACHE_SIZE \
|
| 409 |
+
--data.video-clip.record-build-workers=$VIDEO_RECORD_BUILD_WORKERS \
|
| 410 |
+
--data.video-clip.record-build-chunk-size=$VIDEO_RECORD_BUILD_CHUNK_SIZE \
|
| 411 |
+
$VIDEO_CLIP_TRUST_METADATA_FLAG \
|
| 412 |
+
--data.video-clip.records-cache-dir=$VIDEO_CLIP_RECORDS_CACHE_DIR \
|
| 413 |
+
--data.video-clip.use-omni-hidden-cache \
|
| 414 |
+
--data.video-clip.require-omni-hidden-cache \
|
| 415 |
+
--data.video-clip.omni-hidden-root=$PSI_OMNI_HIDDEN_CACHE_ROOT \
|
| 416 |
+
--data.video-clip.action-exec-horizon=$ACTION_EXEC_HORIZON \
|
| 417 |
+
--data.sampler=task_mixture \
|
| 418 |
+
--data.val-task=text,audio,audio_command,video_clip \
|
| 419 |
+
--data.transform.repack.num-past-frames=$NUM_PAST_FRAMES \
|
| 420 |
+
--data.transform.repack.textop-actions \
|
| 421 |
+
--data.transform.repack.action-chunk-size=$ACTION_CHUNK_SIZE \
|
| 422 |
+
--data.transform.repack.rtc-prefix-frames=$RTC_PREFIX_FRAMES \
|
| 423 |
+
--data.transform.repack.pad-action-dim=39 \
|
| 424 |
+
--data.transform.repack.pad-state-dim=39 \
|
| 425 |
+
--data.transform.field.stat-path=assets/stats/seed_all_g1_motion39_textop_stats.json \
|
| 426 |
+
--data.transform.field.normalize-state \
|
| 427 |
+
--data.transform.field.pad-action-dim=39 \
|
| 428 |
+
--data.transform.field.pad-state-dim=39 \
|
| 429 |
+
--data.transform.model.resize.size 240 320 \
|
| 430 |
+
--data.transform.model.center_crop.size 240 320 \
|
| 431 |
+
--data.transform.model.no-img-aug \
|
| 432 |
+
--data.transform.model.text-only \
|
| 433 |
+
--data.transform.model.audio-only \
|
| 434 |
+
--data.transform.model.video-only \
|
| 435 |
+
--data.transform.model.video-size-pixels=$VIDEO_SIZE_PIXELS \
|
| 436 |
+
--model.model_name_or_path=$PSI_PRETRAIN_BACKBONE_PATH \
|
| 437 |
+
--model.noise-scheduler=flow \
|
| 438 |
+
--model.n_conditions=0 \
|
| 439 |
+
--model.action-chunk-size=$ACTION_CHUNK_SIZE \
|
| 440 |
+
--model.action-dim=39 \
|
| 441 |
+
--model.action-exec-horizon=$ACTION_EXEC_HORIZON \
|
| 442 |
+
--model.observation-horizon=$STATE_HORIZON \
|
| 443 |
+
--model.odim=39 \
|
| 444 |
+
--model.view_feature_dim=2048 \
|
| 445 |
+
--model.use-future-progress-embedding \
|
| 446 |
+
--model.future-progress-dim=$PROGRESS_DIM \
|
| 447 |
+
--model.no-use_film \
|
| 448 |
+
--model.no-combined_temb \
|
| 449 |
+
--model.rtc \
|
| 450 |
+
--model.max-delay=11 \
|
| 451 |
+
--train.max-grad-value=1 \
|
| 452 |
+
--train.loss-group-weight.joint-abs=1.0 \
|
| 453 |
+
--train.loss-group-weight.root-delta-xy=3.0 \
|
| 454 |
+
--train.loss-group-weight.root-rot5-delta=10.0 \
|
| 455 |
+
--train.loss-group-weight.root-height=15.0 \
|
| 456 |
+
--train.loss-group-weight.foot-contact=0.1
|
| 457 |
+
"
|
| 458 |
+
torchrun --nproc_per_node=$NPROC_PER_NODE --master_port=$MASTER_PORT scripts/train.py \
|
| 459 |
+
$args "$@"
|
posttrain-v1-psi0-all-7task-tune-vlm-direct-dense-actionhead-L.sh
ADDED
|
@@ -0,0 +1,313 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
pkill -f monitor.py
|
| 2 |
+
|
| 3 |
+
PROJECT_ROOT=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
|
| 4 |
+
cd "$PROJECT_ROOT"
|
| 5 |
+
export PYTHONPATH="$PROJECT_ROOT/src${PYTHONPATH:+:$PYTHONPATH}"
|
| 6 |
+
export CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7
|
| 7 |
+
export PSI_OUTPUT_DIR=${PSI_OUTPUT_DIR:-$PROJECT_ROOT/runs}
|
| 8 |
+
export PSI_LOG_REPORT_TO=wandb
|
| 9 |
+
|
| 10 |
+
source "$PROJECT_ROOT/scripts/activate_psi_env.sh"
|
| 11 |
+
[ -f .env ] && source .env
|
| 12 |
+
|
| 13 |
+
|
| 14 |
+
export OMP_NUM_THREADS=8
|
| 15 |
+
export TF_CPP_MIN_LOG_LEVEL=3
|
| 16 |
+
|
| 17 |
+
# Keep JSON sources readable in this script, but pass compact JSON to tyro.
|
| 18 |
+
compact_json() {
|
| 19 |
+
python -c 'import json, sys; print(json.dumps(json.load(sys.stdin), separators=(",", ":")))'
|
| 20 |
+
}
|
| 21 |
+
|
| 22 |
+
PSI_SEED_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/Psi0/dataset/seed_all
|
| 23 |
+
PSI_SEED_METADATA=metadata/seed_metadata_v004.csv
|
| 24 |
+
|
| 25 |
+
PSI_DANCE_MOTION_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/GR00T-WholeBodyControl/gear_sonic_deploy/reference/converted/psi0_dance/gvhmr
|
| 26 |
+
PSI_DANCE_AUDIO_VIDEO_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/dance_1723_video_psi0/dance_1723_video_10s
|
| 27 |
+
PSI_DANCE_VIDEO_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/dance_1723_video_psi0/dance_1723_video_10s
|
| 28 |
+
PSI_DANCE_VIDEO_EMBED_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/dance_1723_video_psi0/dance_1723_video_10s_embeds
|
| 29 |
+
PSI_VIDEO_DEMO_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/videogen/motion_example_0506_demo
|
| 30 |
+
PSI_QWEN3_TTS_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/qwen3-tts
|
| 31 |
+
PSI_VIDEO_CLIP_DATASET_ROOT=${PSI_VIDEO_CLIP_DATASET_ROOT:-/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/videogen/psi0_video_clip_50f_dense_stride1_direct_full}
|
| 32 |
+
PSI_DANCE_VIDEO_CLIP_ROOT=$PSI_VIDEO_CLIP_DATASET_ROOT/dance_1723_video_10s
|
| 33 |
+
PSI_MOTION_EXAMPLE_VIDEO_CLIP_ROOT=$PSI_VIDEO_CLIP_DATASET_ROOT/motion_example_0506_demo
|
| 34 |
+
PSI_DANCE_VIDEO_CLIP_EMBED_ROOT=${PSI_DANCE_VIDEO_CLIP_EMBED_ROOT:-$PSI_DANCE_VIDEO_CLIP_ROOT/embeds}
|
| 35 |
+
PSI_MOTION_EXAMPLE_VIDEO_CLIP_EMBED_ROOT=${PSI_MOTION_EXAMPLE_VIDEO_CLIP_EMBED_ROOT:-$PSI_MOTION_EXAMPLE_VIDEO_CLIP_ROOT/embeds}
|
| 36 |
+
PSI_MOTION_EXAMPLE_VIDEO_EMBED_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/videogen/motion_example_0506_demo_video_embeds
|
| 37 |
+
|
| 38 |
+
if [ -z "$PSI_AUDIO_SOURCES_JSON" ]; then
|
| 39 |
+
PSI_AUDIO_SOURCES_JSON=$(compact_json <<JSON
|
| 40 |
+
[
|
| 41 |
+
{
|
| 42 |
+
"name": "psi0_dance_audio",
|
| 43 |
+
"root_dir": "$PSI_DANCE_MOTION_ROOT",
|
| 44 |
+
"video_dir": "$PSI_DANCE_AUDIO_VIDEO_ROOT",
|
| 45 |
+
"dataset_name": "seed-audio-g1"
|
| 46 |
+
}
|
| 47 |
+
]
|
| 48 |
+
JSON
|
| 49 |
+
)
|
| 50 |
+
fi
|
| 51 |
+
|
| 52 |
+
if [ -z "$PSI_AUDIO_COMMAND_SOURCES_JSON" ]; then
|
| 53 |
+
PSI_AUDIO_COMMAND_SOURCES_JSON=$(compact_json <<JSON
|
| 54 |
+
[
|
| 55 |
+
{
|
| 56 |
+
"name": "qwen3_tts_seed_audio_command",
|
| 57 |
+
"root_dir": "$PSI_SEED_ROOT",
|
| 58 |
+
"dataset_root": "$PSI_QWEN3_TTS_ROOT",
|
| 59 |
+
"metadata_jsonl": "metadata.jsonl",
|
| 60 |
+
"motion_dir_key": "filename",
|
| 61 |
+
"audio_path_key": "audio_path",
|
| 62 |
+
"instruction_key": "content_short_description",
|
| 63 |
+
"motion_metadata_csv": "$PSI_SEED_METADATA",
|
| 64 |
+
"motion_metadata_key": "filename",
|
| 65 |
+
"motion_metadata_path_key": "move_g1_path",
|
| 66 |
+
"audio_file_extensions": ".wav",
|
| 67 |
+
"audio_task": "command",
|
| 68 |
+
"dataset_name": "seed-audio-command-g1"
|
| 69 |
+
}
|
| 70 |
+
]
|
| 71 |
+
JSON
|
| 72 |
+
)
|
| 73 |
+
fi
|
| 74 |
+
|
| 75 |
+
if [ -z "$PSI_VIDEO_SOURCES_JSON" ]; then
|
| 76 |
+
PSI_VIDEO_SOURCES_JSON=$(compact_json <<JSON
|
| 77 |
+
[
|
| 78 |
+
{
|
| 79 |
+
"name": "psi0_dance_gvhmr",
|
| 80 |
+
"root_dir": "$PSI_DANCE_MOTION_ROOT",
|
| 81 |
+
"video_dir": "$PSI_DANCE_VIDEO_ROOT",
|
| 82 |
+
"video_embed_dir": "$PSI_DANCE_VIDEO_EMBED_ROOT",
|
| 83 |
+
"dataset_name": "seed-video-g1"
|
| 84 |
+
},
|
| 85 |
+
{
|
| 86 |
+
"name": "motion_example_0506_demo",
|
| 87 |
+
"dataset_root": "$PSI_VIDEO_DEMO_ROOT",
|
| 88 |
+
"metadata_jsonl": "metadata.jsonl",
|
| 89 |
+
"motion_dir_key": "converted_path",
|
| 90 |
+
"video_path_key": "video_path",
|
| 91 |
+
"instruction_key": "video_caption",
|
| 92 |
+
"video_embed_dir": "$PSI_MOTION_EXAMPLE_VIDEO_EMBED_ROOT",
|
| 93 |
+
"dataset_name": "motion-example-video-g1"
|
| 94 |
+
}
|
| 95 |
+
]
|
| 96 |
+
JSON
|
| 97 |
+
)
|
| 98 |
+
fi
|
| 99 |
+
|
| 100 |
+
if [ -z "$PSI_VIDEO_CLIP_SOURCES_JSON" ]; then
|
| 101 |
+
PSI_VIDEO_CLIP_SOURCES_JSON=$(compact_json <<JSON
|
| 102 |
+
[
|
| 103 |
+
{
|
| 104 |
+
"name": "dance_1723_video_10s_clip50",
|
| 105 |
+
"dataset_root": "$PSI_DANCE_VIDEO_CLIP_ROOT",
|
| 106 |
+
"metadata_jsonl": "metadata.jsonl",
|
| 107 |
+
"motion_dir_key": "converted_path",
|
| 108 |
+
"video_path_key": "video_path",
|
| 109 |
+
"instruction_key": "video_caption",
|
| 110 |
+
"video_embed_dir": "$PSI_DANCE_VIDEO_CLIP_EMBED_ROOT",
|
| 111 |
+
"dataset_name": "dance-video-clip-g1"
|
| 112 |
+
},
|
| 113 |
+
{
|
| 114 |
+
"name": "motion_example_0506_demo_clip50",
|
| 115 |
+
"dataset_root": "$PSI_MOTION_EXAMPLE_VIDEO_CLIP_ROOT",
|
| 116 |
+
"metadata_jsonl": "metadata.jsonl",
|
| 117 |
+
"motion_dir_key": "converted_path",
|
| 118 |
+
"video_path_key": "video_path",
|
| 119 |
+
"instruction_key": "video_caption",
|
| 120 |
+
"video_embed_dir": "$PSI_MOTION_EXAMPLE_VIDEO_CLIP_EMBED_ROOT",
|
| 121 |
+
"dataset_name": "motion-example-video-clip-g1"
|
| 122 |
+
}
|
| 123 |
+
]
|
| 124 |
+
JSON
|
| 125 |
+
)
|
| 126 |
+
fi
|
| 127 |
+
|
| 128 |
+
# 7-task from-scratch mix, excluding legacy VIDEO_TEXT / VIDEO_ONLY.
|
| 129 |
+
# Smaller data sources get larger weights. These use a tempered inverse-size
|
| 130 |
+
# rule, (47M / source_size)^0.25, so rare modalities are upweighted without
|
| 131 |
+
# completely starving text/video_clip in BatchMixtureSampler.
|
| 132 |
+
# text stride1 frame samples: ~47,000,000
|
| 133 |
+
# music audio dance frame samples: 213,731
|
| 134 |
+
# audio_command wav files: 41,760
|
| 135 |
+
# dense video_clip metadata: 4,597,576
|
| 136 |
+
PSI_TEXT_TASK_WEIGHT=1.0
|
| 137 |
+
PSI_AUDIO_TEXT_TASK_WEIGHT=3.8509
|
| 138 |
+
PSI_AUDIO_ONLY_TASK_WEIGHT=3.8509
|
| 139 |
+
PSI_AUDIO_COMMAND_TEXT_TASK_WEIGHT=5.7921
|
| 140 |
+
PSI_AUDIO_COMMAND_ONLY_TASK_WEIGHT=5.7921
|
| 141 |
+
PSI_VIDEO_TEXT_TASK_WEIGHT=0
|
| 142 |
+
PSI_VIDEO_ONLY_TASK_WEIGHT=0
|
| 143 |
+
PSI_VIDEO_CLIP_TEXT_TASK_WEIGHT=1.7881
|
| 144 |
+
PSI_VIDEO_CLIP_ONLY_TASK_WEIGHT=1.7881
|
| 145 |
+
|
| 146 |
+
PSI_PRETRAIN_BACKBONE_PATH=/cpfs01/VideoGen/public_ckpt/Qwen/Qwen2.5-Omni-3B
|
| 147 |
+
NUM_PAST_FRAMES=0
|
| 148 |
+
STATE_HORIZON=$((NUM_PAST_FRAMES + 1))
|
| 149 |
+
ACTION_CHUNK_SIZE=30
|
| 150 |
+
ACTION_EXEC_HORIZON=20
|
| 151 |
+
RTC_PREFIX_FRAMES=$((ACTION_CHUNK_SIZE - ACTION_EXEC_HORIZON))
|
| 152 |
+
PROGRESS_DIM=5
|
| 153 |
+
ACTION_HIDDEN_DIM=1920
|
| 154 |
+
ACTION_NUM_BLOCKS=28
|
| 155 |
+
ACTION_NHEADS=20
|
| 156 |
+
ACTION_HEAD_DIM=$((ACTION_HIDDEN_DIM / ACTION_NHEADS))
|
| 157 |
+
VIDEO_SAMPLE_FPS=1.0
|
| 158 |
+
VIDEO_CLIP_SAMPLE_FPS=8.0
|
| 159 |
+
VIDEO_SIZE_PIXELS=50176
|
| 160 |
+
VIDEO_DECODER=ffmpeg
|
| 161 |
+
VIDEO_CACHE_SIZE=64
|
| 162 |
+
VIDEO_CLIP_MOTION_CACHE_SIZE=256
|
| 163 |
+
VIDEO_CLIP_TRUST_METADATA=${VIDEO_CLIP_TRUST_METADATA:-1}
|
| 164 |
+
VIDEO_CLIP_RECORDS_CACHE_DIR=${VIDEO_CLIP_RECORDS_CACHE_DIR:-$PSI_VIDEO_CLIP_DATASET_ROOT/.index_cache}
|
| 165 |
+
if [ "$VIDEO_CLIP_TRUST_METADATA" = "1" ] || [ "$VIDEO_CLIP_TRUST_METADATA" = "true" ] || [ "$VIDEO_CLIP_TRUST_METADATA" = "True" ]; then
|
| 166 |
+
VIDEO_CLIP_TRUST_METADATA_FLAG="--data.video-clip.trust-metadata"
|
| 167 |
+
else
|
| 168 |
+
VIDEO_CLIP_TRUST_METADATA_FLAG="--data.video-clip.no-trust-metadata"
|
| 169 |
+
fi
|
| 170 |
+
DATALOADER_NUM_WORKERS=4
|
| 171 |
+
VAL_DATALOADER_NUM_WORKERS=2
|
| 172 |
+
|
| 173 |
+
NPROC_PER_NODE=$(echo $CUDA_VISIBLE_DEVICES | tr ',' '\n' | wc -l)
|
| 174 |
+
|
| 175 |
+
echo "Training with $NPROC_PER_NODE GPU(s): $CUDA_VISIBLE_DEVICES"
|
| 176 |
+
echo "Text root: $PSI_SEED_ROOT"
|
| 177 |
+
echo "Dance motion root: $PSI_DANCE_MOTION_ROOT"
|
| 178 |
+
echo "Dance audio/video root: $PSI_DANCE_AUDIO_VIDEO_ROOT"
|
| 179 |
+
echo "Dance video root: $PSI_DANCE_VIDEO_ROOT"
|
| 180 |
+
echo "Dance video embed root: $PSI_DANCE_VIDEO_EMBED_ROOT"
|
| 181 |
+
echo "Qwen3-TTS acdudio command root: $PSI_QWEN3_TTS_ROOT"
|
| 182 |
+
echo "Audio sources: $PSI_AUDIO_SOURCES_JSON"
|
| 183 |
+
echo "Audio command sources: $PSI_AUDIO_COMMAND_SOURCES_JSON"
|
| 184 |
+
echo "Video sources: $PSI_VIDEO_SOURCES_JSON"
|
| 185 |
+
echo "Video clip sources: $PSI_VIDEO_CLIP_SOURCES_JSON"
|
| 186 |
+
echo "Task weights: text=$PSI_TEXT_TASK_WEIGHT audio_text=$PSI_AUDIO_TEXT_TASK_WEIGHT audio_only=$PSI_AUDIO_ONLY_TASK_WEIGHT audio_command_text=$PSI_AUDIO_COMMAND_TEXT_TASK_WEIGHT audio_command_only=$PSI_AUDIO_COMMAND_ONLY_TASK_WEIGHT video_text=$PSI_VIDEO_TEXT_TASK_WEIGHT video_only=$PSI_VIDEO_ONLY_TASK_WEIGHT video_clip_text=$PSI_VIDEO_CLIP_TEXT_TASK_WEIGHT video_clip_only=$PSI_VIDEO_CLIP_ONLY_TASK_WEIGHT"
|
| 187 |
+
# echo "Resume run dir: $PSI_PRETRAIN_RUN_DIR"
|
| 188 |
+
echo "TextOp 39D action/state enabled"
|
| 189 |
+
echo "Future progress embedding: enabled, dim $PROGRESS_DIM"
|
| 190 |
+
echo "Action head scale: L hidden=$ACTION_HIDDEN_DIM depth=$ACTION_NUM_BLOCKS heads=$ACTION_NHEADS head_dim=$ACTION_HEAD_DIM"
|
| 191 |
+
echo "Backbone path: $PSI_PRETRAIN_BACKBONE_PATH"
|
| 192 |
+
echo "Output dir: $PSI_OUTPUT_DIR"
|
| 193 |
+
echo "Video clip trust metadata: $VIDEO_CLIP_TRUST_METADATA"
|
| 194 |
+
echo "Video clip records cache dir: $VIDEO_CLIP_RECORDS_CACHE_DIR"
|
| 195 |
+
|
| 196 |
+
args="posttrain_v1_seed_text_audio_video_mix_psi0_config \
|
| 197 |
+
--seed=292285 \
|
| 198 |
+
--exp=posttrain_actionhead_L \
|
| 199 |
+
--timestamp=$(date +"%y%m%d%H%M") \
|
| 200 |
+
--train.name=posttrain_v1_trimodal \
|
| 201 |
+
--train.data_parallel=ddp \
|
| 202 |
+
--train.mixed_precision=bf16 \
|
| 203 |
+
--train.train_batch_size=16 \
|
| 204 |
+
--train.val_batch_size=32 \
|
| 205 |
+
--train.num-workers=$DATALOADER_NUM_WORKERS \
|
| 206 |
+
--train.val-num-workers=$VAL_DATALOADER_NUM_WORKERS \
|
| 207 |
+
--train.resume_from_checkpoint=None \
|
| 208 |
+
--train.max_checkpoints_to_keep=1 \
|
| 209 |
+
--train.gradient_accumulation_steps=1 \
|
| 210 |
+
--train.learning_rate=1e-4 \
|
| 211 |
+
--train.max_training_steps=500000 \
|
| 212 |
+
--train.warmup_ratio=None \
|
| 213 |
+
--train.warmup_steps=1000 \
|
| 214 |
+
--train.checkpointing_steps=1000 \
|
| 215 |
+
--train.validation_steps=1000 \
|
| 216 |
+
--train.val_num_batches=20 \
|
| 217 |
+
--train.lr_scheduler_type=cosine_with_min_lr \
|
| 218 |
+
--train.lr_scheduler_kwargs.betas 0.9 0.999 \
|
| 219 |
+
--train.lr_scheduler_kwargs.weight_decay=0 \
|
| 220 |
+
--train.lr_scheduler_kwargs.eps=1e-8 \
|
| 221 |
+
--log.report_to=$PSI_LOG_REPORT_TO \
|
| 222 |
+
--train.output_dir=$PSI_OUTPUT_DIR \
|
| 223 |
+
--data.text.root-dir=$PSI_SEED_ROOT \
|
| 224 |
+
--data.text.metadata-csv=$PSI_SEED_METADATA \
|
| 225 |
+
--data.text-ratio=$PSI_TEXT_TASK_WEIGHT \
|
| 226 |
+
--data.audio.sources-json=$PSI_AUDIO_SOURCES_JSON \
|
| 227 |
+
--data.audio-ratio=$PSI_AUDIO_TEXT_TASK_WEIGHT \
|
| 228 |
+
--data.audio-only-ratio=$PSI_AUDIO_ONLY_TASK_WEIGHT \
|
| 229 |
+
--data.audio.sample-stride=1 \
|
| 230 |
+
--data.audio.motion-fps=50 \
|
| 231 |
+
--data.audio.audio-sampling-rate=16000 \
|
| 232 |
+
--data.audio.audio-window-seconds=1.0 \
|
| 233 |
+
--data.audio.audio-left-context-seconds=0.0 \
|
| 234 |
+
--data.audio.full-audio \
|
| 235 |
+
--data.audio-command.sources-json=$PSI_AUDIO_COMMAND_SOURCES_JSON \
|
| 236 |
+
--data.audio-command-ratio=$PSI_AUDIO_COMMAND_TEXT_TASK_WEIGHT \
|
| 237 |
+
--data.audio-command-only-ratio=$PSI_AUDIO_COMMAND_ONLY_TASK_WEIGHT \
|
| 238 |
+
--data.audio-command.sample-stride=1 \
|
| 239 |
+
--data.audio-command.motion-fps=50 \
|
| 240 |
+
--data.audio-command.audio-sampling-rate=16000 \
|
| 241 |
+
--data.audio-command.audio-window-seconds=1.0 \
|
| 242 |
+
--data.audio-command.audio-left-context-seconds=0.0 \
|
| 243 |
+
--data.audio-command.full-audio \
|
| 244 |
+
--data.video-ratio=$PSI_VIDEO_TEXT_TASK_WEIGHT \
|
| 245 |
+
--data.video-only-ratio=$PSI_VIDEO_ONLY_TASK_WEIGHT \
|
| 246 |
+
--data.video.sample-stride=1 \
|
| 247 |
+
--data.video.motion-fps=50 \
|
| 248 |
+
--data.video.video-sample-fps=$VIDEO_SAMPLE_FPS \
|
| 249 |
+
--data.video.video-size-pixels=$VIDEO_SIZE_PIXELS \
|
| 250 |
+
--data.video.video-decoder=$VIDEO_DECODER \
|
| 251 |
+
--data.video.video-cache-size=$VIDEO_CACHE_SIZE \
|
| 252 |
+
--data.video.sources-json=$PSI_VIDEO_SOURCES_JSON \
|
| 253 |
+
--data.video-clip-text-ratio=$PSI_VIDEO_CLIP_TEXT_TASK_WEIGHT \
|
| 254 |
+
--data.video-clip-only-ratio=$PSI_VIDEO_CLIP_ONLY_TASK_WEIGHT \
|
| 255 |
+
--data.video-clip.sources-json=$PSI_VIDEO_CLIP_SOURCES_JSON \
|
| 256 |
+
--data.video-clip.sample-stride=1 \
|
| 257 |
+
--data.video-clip.motion-fps=50 \
|
| 258 |
+
--data.video-clip.video-sample-fps=$VIDEO_CLIP_SAMPLE_FPS \
|
| 259 |
+
--data.video-clip.video-size-pixels=$VIDEO_SIZE_PIXELS \
|
| 260 |
+
--data.video-clip.video-decoder=$VIDEO_DECODER \
|
| 261 |
+
--data.video-clip.video-cache-size=$VIDEO_CACHE_SIZE \
|
| 262 |
+
--data.video-clip.motion-cache-size=$VIDEO_CLIP_MOTION_CACHE_SIZE \
|
| 263 |
+
$VIDEO_CLIP_TRUST_METADATA_FLAG \
|
| 264 |
+
--data.video-clip.records-cache-dir=$VIDEO_CLIP_RECORDS_CACHE_DIR \
|
| 265 |
+
--data.video-clip.action-exec-horizon=$ACTION_EXEC_HORIZON \
|
| 266 |
+
--data.sampler=batch_mixture \
|
| 267 |
+
--data.val-task=text,audio,audio_only,audio_command,audio_command_only,video_clip,video_clip_only \
|
| 268 |
+
--data.transform.repack.num-past-frames=$NUM_PAST_FRAMES \
|
| 269 |
+
--data.transform.repack.textop-actions \
|
| 270 |
+
--data.transform.repack.action-chunk-size=$ACTION_CHUNK_SIZE \
|
| 271 |
+
--data.transform.repack.rtc-prefix-frames=$RTC_PREFIX_FRAMES \
|
| 272 |
+
--data.transform.repack.pad-action-dim=39 \
|
| 273 |
+
--data.transform.repack.pad-state-dim=39 \
|
| 274 |
+
--data.transform.field.stat-path=assets/stats/seed_all_g1_motion39_textop_stats.json \
|
| 275 |
+
--data.transform.field.normalize-state \
|
| 276 |
+
--data.transform.field.pad-action-dim=39 \
|
| 277 |
+
--data.transform.field.pad-state-dim=39 \
|
| 278 |
+
--data.transform.model.resize.size 240 320 \
|
| 279 |
+
--data.transform.model.center_crop.size 240 320 \
|
| 280 |
+
--data.transform.model.no-img-aug \
|
| 281 |
+
--data.transform.model.text-only \
|
| 282 |
+
--data.transform.model.audio-only \
|
| 283 |
+
--data.transform.model.video-only \
|
| 284 |
+
--data.transform.model.video-size-pixels=$VIDEO_SIZE_PIXELS \
|
| 285 |
+
--model.model_name_or_path=$PSI_PRETRAIN_BACKBONE_PATH \
|
| 286 |
+
--model.noise-scheduler=flow \
|
| 287 |
+
--model.n_conditions=0 \
|
| 288 |
+
--model.action-chunk-size=$ACTION_CHUNK_SIZE \
|
| 289 |
+
--model.action-dim=39 \
|
| 290 |
+
--model.action-exec-horizon=$ACTION_EXEC_HORIZON \
|
| 291 |
+
--model.observation-horizon=$STATE_HORIZON \
|
| 292 |
+
--model.odim=39 \
|
| 293 |
+
--model.view_feature_dim=2048 \
|
| 294 |
+
--model.hidden-dim=$ACTION_HIDDEN_DIM \
|
| 295 |
+
--model.num-blocks=$ACTION_NUM_BLOCKS \
|
| 296 |
+
--model.nhead=$ACTION_NHEADS \
|
| 297 |
+
--model.use-future-progress-embedding \
|
| 298 |
+
--model.future-progress-dim=$PROGRESS_DIM \
|
| 299 |
+
--model.tune-vlm \
|
| 300 |
+
--model.lang-backbone-lr=1e-5 \
|
| 301 |
+
--model.no-use_film \
|
| 302 |
+
--model.no-combined_temb \
|
| 303 |
+
--model.rtc \
|
| 304 |
+
--model.max-delay=11 \
|
| 305 |
+
--train.max-grad-value=1 \
|
| 306 |
+
--train.loss-group-weight.joint-abs=1.0 \
|
| 307 |
+
--train.loss-group-weight.root-delta-xy=3.0 \
|
| 308 |
+
--train.loss-group-weight.root-rot5-delta=10.0 \
|
| 309 |
+
--train.loss-group-weight.root-height=15.0 \
|
| 310 |
+
--train.loss-group-weight.foot-contact=0.1
|
| 311 |
+
"
|
| 312 |
+
torchrun --nproc_per_node=$NPROC_PER_NODE --master_port=29500 scripts/train.py \
|
| 313 |
+
$args "$@"
|
posttrain-v1-psi0-all-7task-tune-vlm-direct-dense-actionhead-M.sh
ADDED
|
@@ -0,0 +1,313 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
pkill -f monitor.py
|
| 2 |
+
|
| 3 |
+
PROJECT_ROOT=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
|
| 4 |
+
cd "$PROJECT_ROOT"
|
| 5 |
+
export PYTHONPATH="$PROJECT_ROOT/src${PYTHONPATH:+:$PYTHONPATH}"
|
| 6 |
+
export CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7
|
| 7 |
+
export PSI_OUTPUT_DIR=${PSI_OUTPUT_DIR:-$PROJECT_ROOT/runs}
|
| 8 |
+
export PSI_LOG_REPORT_TO=wandb
|
| 9 |
+
|
| 10 |
+
source "$PROJECT_ROOT/scripts/activate_psi_env.sh"
|
| 11 |
+
[ -f .env ] && source .env
|
| 12 |
+
|
| 13 |
+
|
| 14 |
+
export OMP_NUM_THREADS=8
|
| 15 |
+
export TF_CPP_MIN_LOG_LEVEL=3
|
| 16 |
+
|
| 17 |
+
# Keep JSON sources readable in this script, but pass compact JSON to tyro.
|
| 18 |
+
compact_json() {
|
| 19 |
+
python -c 'import json, sys; print(json.dumps(json.load(sys.stdin), separators=(",", ":")))'
|
| 20 |
+
}
|
| 21 |
+
|
| 22 |
+
PSI_SEED_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/Psi0/dataset/seed_all
|
| 23 |
+
PSI_SEED_METADATA=metadata/seed_metadata_v004.csv
|
| 24 |
+
|
| 25 |
+
PSI_DANCE_MOTION_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/GR00T-WholeBodyControl/gear_sonic_deploy/reference/converted/psi0_dance/gvhmr
|
| 26 |
+
PSI_DANCE_AUDIO_VIDEO_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/dance_1723_video_psi0/dance_1723_video_10s
|
| 27 |
+
PSI_DANCE_VIDEO_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/dance_1723_video_psi0/dance_1723_video_10s
|
| 28 |
+
PSI_DANCE_VIDEO_EMBED_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/dance_1723_video_psi0/dance_1723_video_10s_embeds
|
| 29 |
+
PSI_VIDEO_DEMO_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/videogen/motion_example_0506_demo
|
| 30 |
+
PSI_QWEN3_TTS_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/qwen3-tts
|
| 31 |
+
PSI_VIDEO_CLIP_DATASET_ROOT=${PSI_VIDEO_CLIP_DATASET_ROOT:-/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/videogen/psi0_video_clip_50f_dense_stride1_direct_full}
|
| 32 |
+
PSI_DANCE_VIDEO_CLIP_ROOT=$PSI_VIDEO_CLIP_DATASET_ROOT/dance_1723_video_10s
|
| 33 |
+
PSI_MOTION_EXAMPLE_VIDEO_CLIP_ROOT=$PSI_VIDEO_CLIP_DATASET_ROOT/motion_example_0506_demo
|
| 34 |
+
PSI_DANCE_VIDEO_CLIP_EMBED_ROOT=${PSI_DANCE_VIDEO_CLIP_EMBED_ROOT:-$PSI_DANCE_VIDEO_CLIP_ROOT/embeds}
|
| 35 |
+
PSI_MOTION_EXAMPLE_VIDEO_CLIP_EMBED_ROOT=${PSI_MOTION_EXAMPLE_VIDEO_CLIP_EMBED_ROOT:-$PSI_MOTION_EXAMPLE_VIDEO_CLIP_ROOT/embeds}
|
| 36 |
+
PSI_MOTION_EXAMPLE_VIDEO_EMBED_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/videogen/motion_example_0506_demo_video_embeds
|
| 37 |
+
|
| 38 |
+
if [ -z "$PSI_AUDIO_SOURCES_JSON" ]; then
|
| 39 |
+
PSI_AUDIO_SOURCES_JSON=$(compact_json <<JSON
|
| 40 |
+
[
|
| 41 |
+
{
|
| 42 |
+
"name": "psi0_dance_audio",
|
| 43 |
+
"root_dir": "$PSI_DANCE_MOTION_ROOT",
|
| 44 |
+
"video_dir": "$PSI_DANCE_AUDIO_VIDEO_ROOT",
|
| 45 |
+
"dataset_name": "seed-audio-g1"
|
| 46 |
+
}
|
| 47 |
+
]
|
| 48 |
+
JSON
|
| 49 |
+
)
|
| 50 |
+
fi
|
| 51 |
+
|
| 52 |
+
if [ -z "$PSI_AUDIO_COMMAND_SOURCES_JSON" ]; then
|
| 53 |
+
PSI_AUDIO_COMMAND_SOURCES_JSON=$(compact_json <<JSON
|
| 54 |
+
[
|
| 55 |
+
{
|
| 56 |
+
"name": "qwen3_tts_seed_audio_command",
|
| 57 |
+
"root_dir": "$PSI_SEED_ROOT",
|
| 58 |
+
"dataset_root": "$PSI_QWEN3_TTS_ROOT",
|
| 59 |
+
"metadata_jsonl": "metadata.jsonl",
|
| 60 |
+
"motion_dir_key": "filename",
|
| 61 |
+
"audio_path_key": "audio_path",
|
| 62 |
+
"instruction_key": "content_short_description",
|
| 63 |
+
"motion_metadata_csv": "$PSI_SEED_METADATA",
|
| 64 |
+
"motion_metadata_key": "filename",
|
| 65 |
+
"motion_metadata_path_key": "move_g1_path",
|
| 66 |
+
"audio_file_extensions": ".wav",
|
| 67 |
+
"audio_task": "command",
|
| 68 |
+
"dataset_name": "seed-audio-command-g1"
|
| 69 |
+
}
|
| 70 |
+
]
|
| 71 |
+
JSON
|
| 72 |
+
)
|
| 73 |
+
fi
|
| 74 |
+
|
| 75 |
+
if [ -z "$PSI_VIDEO_SOURCES_JSON" ]; then
|
| 76 |
+
PSI_VIDEO_SOURCES_JSON=$(compact_json <<JSON
|
| 77 |
+
[
|
| 78 |
+
{
|
| 79 |
+
"name": "psi0_dance_gvhmr",
|
| 80 |
+
"root_dir": "$PSI_DANCE_MOTION_ROOT",
|
| 81 |
+
"video_dir": "$PSI_DANCE_VIDEO_ROOT",
|
| 82 |
+
"video_embed_dir": "$PSI_DANCE_VIDEO_EMBED_ROOT",
|
| 83 |
+
"dataset_name": "seed-video-g1"
|
| 84 |
+
},
|
| 85 |
+
{
|
| 86 |
+
"name": "motion_example_0506_demo",
|
| 87 |
+
"dataset_root": "$PSI_VIDEO_DEMO_ROOT",
|
| 88 |
+
"metadata_jsonl": "metadata.jsonl",
|
| 89 |
+
"motion_dir_key": "converted_path",
|
| 90 |
+
"video_path_key": "video_path",
|
| 91 |
+
"instruction_key": "video_caption",
|
| 92 |
+
"video_embed_dir": "$PSI_MOTION_EXAMPLE_VIDEO_EMBED_ROOT",
|
| 93 |
+
"dataset_name": "motion-example-video-g1"
|
| 94 |
+
}
|
| 95 |
+
]
|
| 96 |
+
JSON
|
| 97 |
+
)
|
| 98 |
+
fi
|
| 99 |
+
|
| 100 |
+
if [ -z "$PSI_VIDEO_CLIP_SOURCES_JSON" ]; then
|
| 101 |
+
PSI_VIDEO_CLIP_SOURCES_JSON=$(compact_json <<JSON
|
| 102 |
+
[
|
| 103 |
+
{
|
| 104 |
+
"name": "dance_1723_video_10s_clip50",
|
| 105 |
+
"dataset_root": "$PSI_DANCE_VIDEO_CLIP_ROOT",
|
| 106 |
+
"metadata_jsonl": "metadata.jsonl",
|
| 107 |
+
"motion_dir_key": "converted_path",
|
| 108 |
+
"video_path_key": "video_path",
|
| 109 |
+
"instruction_key": "video_caption",
|
| 110 |
+
"video_embed_dir": "$PSI_DANCE_VIDEO_CLIP_EMBED_ROOT",
|
| 111 |
+
"dataset_name": "dance-video-clip-g1"
|
| 112 |
+
},
|
| 113 |
+
{
|
| 114 |
+
"name": "motion_example_0506_demo_clip50",
|
| 115 |
+
"dataset_root": "$PSI_MOTION_EXAMPLE_VIDEO_CLIP_ROOT",
|
| 116 |
+
"metadata_jsonl": "metadata.jsonl",
|
| 117 |
+
"motion_dir_key": "converted_path",
|
| 118 |
+
"video_path_key": "video_path",
|
| 119 |
+
"instruction_key": "video_caption",
|
| 120 |
+
"video_embed_dir": "$PSI_MOTION_EXAMPLE_VIDEO_CLIP_EMBED_ROOT",
|
| 121 |
+
"dataset_name": "motion-example-video-clip-g1"
|
| 122 |
+
}
|
| 123 |
+
]
|
| 124 |
+
JSON
|
| 125 |
+
)
|
| 126 |
+
fi
|
| 127 |
+
|
| 128 |
+
# 7-task from-scratch mix, excluding legacy VIDEO_TEXT / VIDEO_ONLY.
|
| 129 |
+
# Smaller data sources get larger weights. These use a tempered inverse-size
|
| 130 |
+
# rule, (47M / source_size)^0.25, so rare modalities are upweighted without
|
| 131 |
+
# completely starving text/video_clip in BatchMixtureSampler.
|
| 132 |
+
# text stride1 frame samples: ~47,000,000
|
| 133 |
+
# music audio dance frame samples: 213,731
|
| 134 |
+
# audio_command wav files: 41,760
|
| 135 |
+
# dense video_clip metadata: 4,597,576
|
| 136 |
+
PSI_TEXT_TASK_WEIGHT=1.0
|
| 137 |
+
PSI_AUDIO_TEXT_TASK_WEIGHT=3.8509
|
| 138 |
+
PSI_AUDIO_ONLY_TASK_WEIGHT=3.8509
|
| 139 |
+
PSI_AUDIO_COMMAND_TEXT_TASK_WEIGHT=5.7921
|
| 140 |
+
PSI_AUDIO_COMMAND_ONLY_TASK_WEIGHT=5.7921
|
| 141 |
+
PSI_VIDEO_TEXT_TASK_WEIGHT=0
|
| 142 |
+
PSI_VIDEO_ONLY_TASK_WEIGHT=0
|
| 143 |
+
PSI_VIDEO_CLIP_TEXT_TASK_WEIGHT=1.7881
|
| 144 |
+
PSI_VIDEO_CLIP_ONLY_TASK_WEIGHT=1.7881
|
| 145 |
+
|
| 146 |
+
PSI_PRETRAIN_BACKBONE_PATH=/cpfs01/VideoGen/public_ckpt/Qwen/Qwen2.5-Omni-3B
|
| 147 |
+
NUM_PAST_FRAMES=0
|
| 148 |
+
STATE_HORIZON=$((NUM_PAST_FRAMES + 1))
|
| 149 |
+
ACTION_CHUNK_SIZE=30
|
| 150 |
+
ACTION_EXEC_HORIZON=20
|
| 151 |
+
RTC_PREFIX_FRAMES=$((ACTION_CHUNK_SIZE - ACTION_EXEC_HORIZON))
|
| 152 |
+
PROGRESS_DIM=5
|
| 153 |
+
ACTION_HIDDEN_DIM=1152
|
| 154 |
+
ACTION_NUM_BLOCKS=16
|
| 155 |
+
ACTION_NHEADS=12
|
| 156 |
+
ACTION_HEAD_DIM=$((ACTION_HIDDEN_DIM / ACTION_NHEADS))
|
| 157 |
+
VIDEO_SAMPLE_FPS=1.0
|
| 158 |
+
VIDEO_CLIP_SAMPLE_FPS=8.0
|
| 159 |
+
VIDEO_SIZE_PIXELS=50176
|
| 160 |
+
VIDEO_DECODER=ffmpeg
|
| 161 |
+
VIDEO_CACHE_SIZE=64
|
| 162 |
+
VIDEO_CLIP_MOTION_CACHE_SIZE=256
|
| 163 |
+
VIDEO_CLIP_TRUST_METADATA=${VIDEO_CLIP_TRUST_METADATA:-1}
|
| 164 |
+
VIDEO_CLIP_RECORDS_CACHE_DIR=${VIDEO_CLIP_RECORDS_CACHE_DIR:-$PSI_VIDEO_CLIP_DATASET_ROOT/.index_cache}
|
| 165 |
+
if [ "$VIDEO_CLIP_TRUST_METADATA" = "1" ] || [ "$VIDEO_CLIP_TRUST_METADATA" = "true" ] || [ "$VIDEO_CLIP_TRUST_METADATA" = "True" ]; then
|
| 166 |
+
VIDEO_CLIP_TRUST_METADATA_FLAG="--data.video-clip.trust-metadata"
|
| 167 |
+
else
|
| 168 |
+
VIDEO_CLIP_TRUST_METADATA_FLAG="--data.video-clip.no-trust-metadata"
|
| 169 |
+
fi
|
| 170 |
+
DATALOADER_NUM_WORKERS=4
|
| 171 |
+
VAL_DATALOADER_NUM_WORKERS=2
|
| 172 |
+
|
| 173 |
+
NPROC_PER_NODE=$(echo $CUDA_VISIBLE_DEVICES | tr ',' '\n' | wc -l)
|
| 174 |
+
|
| 175 |
+
echo "Training with $NPROC_PER_NODE GPU(s): $CUDA_VISIBLE_DEVICES"
|
| 176 |
+
echo "Text root: $PSI_SEED_ROOT"
|
| 177 |
+
echo "Dance motion root: $PSI_DANCE_MOTION_ROOT"
|
| 178 |
+
echo "Dance audio/video root: $PSI_DANCE_AUDIO_VIDEO_ROOT"
|
| 179 |
+
echo "Dance video root: $PSI_DANCE_VIDEO_ROOT"
|
| 180 |
+
echo "Dance video embed root: $PSI_DANCE_VIDEO_EMBED_ROOT"
|
| 181 |
+
echo "Qwen3-TTS acdudio command root: $PSI_QWEN3_TTS_ROOT"
|
| 182 |
+
echo "Audio sources: $PSI_AUDIO_SOURCES_JSON"
|
| 183 |
+
echo "Audio command sources: $PSI_AUDIO_COMMAND_SOURCES_JSON"
|
| 184 |
+
echo "Video sources: $PSI_VIDEO_SOURCES_JSON"
|
| 185 |
+
echo "Video clip sources: $PSI_VIDEO_CLIP_SOURCES_JSON"
|
| 186 |
+
echo "Task weights: text=$PSI_TEXT_TASK_WEIGHT audio_text=$PSI_AUDIO_TEXT_TASK_WEIGHT audio_only=$PSI_AUDIO_ONLY_TASK_WEIGHT audio_command_text=$PSI_AUDIO_COMMAND_TEXT_TASK_WEIGHT audio_command_only=$PSI_AUDIO_COMMAND_ONLY_TASK_WEIGHT video_text=$PSI_VIDEO_TEXT_TASK_WEIGHT video_only=$PSI_VIDEO_ONLY_TASK_WEIGHT video_clip_text=$PSI_VIDEO_CLIP_TEXT_TASK_WEIGHT video_clip_only=$PSI_VIDEO_CLIP_ONLY_TASK_WEIGHT"
|
| 187 |
+
# echo "Resume run dir: $PSI_PRETRAIN_RUN_DIR"
|
| 188 |
+
echo "TextOp 39D action/state enabled"
|
| 189 |
+
echo "Future progress embedding: enabled, dim $PROGRESS_DIM"
|
| 190 |
+
echo "Action head scale: M hidden=$ACTION_HIDDEN_DIM depth=$ACTION_NUM_BLOCKS heads=$ACTION_NHEADS head_dim=$ACTION_HEAD_DIM"
|
| 191 |
+
echo "Backbone path: $PSI_PRETRAIN_BACKBONE_PATH"
|
| 192 |
+
echo "Output dir: $PSI_OUTPUT_DIR"
|
| 193 |
+
echo "Video clip trust metadata: $VIDEO_CLIP_TRUST_METADATA"
|
| 194 |
+
echo "Video clip records cache dir: $VIDEO_CLIP_RECORDS_CACHE_DIR"
|
| 195 |
+
|
| 196 |
+
args="posttrain_v1_seed_text_audio_video_mix_psi0_config \
|
| 197 |
+
--seed=292285 \
|
| 198 |
+
--exp=posttrain_actionhead_M \
|
| 199 |
+
--timestamp=$(date +"%y%m%d%H%M") \
|
| 200 |
+
--train.name=posttrain_v1_trimodal \
|
| 201 |
+
--train.data_parallel=ddp \
|
| 202 |
+
--train.mixed_precision=bf16 \
|
| 203 |
+
--train.train_batch_size=32 \
|
| 204 |
+
--train.val_batch_size=64 \
|
| 205 |
+
--train.num-workers=$DATALOADER_NUM_WORKERS \
|
| 206 |
+
--train.val-num-workers=$VAL_DATALOADER_NUM_WORKERS \
|
| 207 |
+
--train.resume_from_checkpoint=None \
|
| 208 |
+
--train.max_checkpoints_to_keep=1 \
|
| 209 |
+
--train.gradient_accumulation_steps=1 \
|
| 210 |
+
--train.learning_rate=1e-4 \
|
| 211 |
+
--train.max_training_steps=500000 \
|
| 212 |
+
--train.warmup_ratio=None \
|
| 213 |
+
--train.warmup_steps=1000 \
|
| 214 |
+
--train.checkpointing_steps=1000 \
|
| 215 |
+
--train.validation_steps=1000 \
|
| 216 |
+
--train.val_num_batches=20 \
|
| 217 |
+
--train.lr_scheduler_type=cosine_with_min_lr \
|
| 218 |
+
--train.lr_scheduler_kwargs.betas 0.9 0.999 \
|
| 219 |
+
--train.lr_scheduler_kwargs.weight_decay=0 \
|
| 220 |
+
--train.lr_scheduler_kwargs.eps=1e-8 \
|
| 221 |
+
--log.report_to=$PSI_LOG_REPORT_TO \
|
| 222 |
+
--train.output_dir=$PSI_OUTPUT_DIR \
|
| 223 |
+
--data.text.root-dir=$PSI_SEED_ROOT \
|
| 224 |
+
--data.text.metadata-csv=$PSI_SEED_METADATA \
|
| 225 |
+
--data.text-ratio=$PSI_TEXT_TASK_WEIGHT \
|
| 226 |
+
--data.audio.sources-json=$PSI_AUDIO_SOURCES_JSON \
|
| 227 |
+
--data.audio-ratio=$PSI_AUDIO_TEXT_TASK_WEIGHT \
|
| 228 |
+
--data.audio-only-ratio=$PSI_AUDIO_ONLY_TASK_WEIGHT \
|
| 229 |
+
--data.audio.sample-stride=1 \
|
| 230 |
+
--data.audio.motion-fps=50 \
|
| 231 |
+
--data.audio.audio-sampling-rate=16000 \
|
| 232 |
+
--data.audio.audio-window-seconds=1.0 \
|
| 233 |
+
--data.audio.audio-left-context-seconds=0.0 \
|
| 234 |
+
--data.audio.full-audio \
|
| 235 |
+
--data.audio-command.sources-json=$PSI_AUDIO_COMMAND_SOURCES_JSON \
|
| 236 |
+
--data.audio-command-ratio=$PSI_AUDIO_COMMAND_TEXT_TASK_WEIGHT \
|
| 237 |
+
--data.audio-command-only-ratio=$PSI_AUDIO_COMMAND_ONLY_TASK_WEIGHT \
|
| 238 |
+
--data.audio-command.sample-stride=1 \
|
| 239 |
+
--data.audio-command.motion-fps=50 \
|
| 240 |
+
--data.audio-command.audio-sampling-rate=16000 \
|
| 241 |
+
--data.audio-command.audio-window-seconds=1.0 \
|
| 242 |
+
--data.audio-command.audio-left-context-seconds=0.0 \
|
| 243 |
+
--data.audio-command.full-audio \
|
| 244 |
+
--data.video-ratio=$PSI_VIDEO_TEXT_TASK_WEIGHT \
|
| 245 |
+
--data.video-only-ratio=$PSI_VIDEO_ONLY_TASK_WEIGHT \
|
| 246 |
+
--data.video.sample-stride=1 \
|
| 247 |
+
--data.video.motion-fps=50 \
|
| 248 |
+
--data.video.video-sample-fps=$VIDEO_SAMPLE_FPS \
|
| 249 |
+
--data.video.video-size-pixels=$VIDEO_SIZE_PIXELS \
|
| 250 |
+
--data.video.video-decoder=$VIDEO_DECODER \
|
| 251 |
+
--data.video.video-cache-size=$VIDEO_CACHE_SIZE \
|
| 252 |
+
--data.video.sources-json=$PSI_VIDEO_SOURCES_JSON \
|
| 253 |
+
--data.video-clip-text-ratio=$PSI_VIDEO_CLIP_TEXT_TASK_WEIGHT \
|
| 254 |
+
--data.video-clip-only-ratio=$PSI_VIDEO_CLIP_ONLY_TASK_WEIGHT \
|
| 255 |
+
--data.video-clip.sources-json=$PSI_VIDEO_CLIP_SOURCES_JSON \
|
| 256 |
+
--data.video-clip.sample-stride=1 \
|
| 257 |
+
--data.video-clip.motion-fps=50 \
|
| 258 |
+
--data.video-clip.video-sample-fps=$VIDEO_CLIP_SAMPLE_FPS \
|
| 259 |
+
--data.video-clip.video-size-pixels=$VIDEO_SIZE_PIXELS \
|
| 260 |
+
--data.video-clip.video-decoder=$VIDEO_DECODER \
|
| 261 |
+
--data.video-clip.video-cache-size=$VIDEO_CACHE_SIZE \
|
| 262 |
+
--data.video-clip.motion-cache-size=$VIDEO_CLIP_MOTION_CACHE_SIZE \
|
| 263 |
+
$VIDEO_CLIP_TRUST_METADATA_FLAG \
|
| 264 |
+
--data.video-clip.records-cache-dir=$VIDEO_CLIP_RECORDS_CACHE_DIR \
|
| 265 |
+
--data.video-clip.action-exec-horizon=$ACTION_EXEC_HORIZON \
|
| 266 |
+
--data.sampler=batch_mixture \
|
| 267 |
+
--data.val-task=text,audio,audio_only,audio_command,audio_command_only,video_clip,video_clip_only \
|
| 268 |
+
--data.transform.repack.num-past-frames=$NUM_PAST_FRAMES \
|
| 269 |
+
--data.transform.repack.textop-actions \
|
| 270 |
+
--data.transform.repack.action-chunk-size=$ACTION_CHUNK_SIZE \
|
| 271 |
+
--data.transform.repack.rtc-prefix-frames=$RTC_PREFIX_FRAMES \
|
| 272 |
+
--data.transform.repack.pad-action-dim=39 \
|
| 273 |
+
--data.transform.repack.pad-state-dim=39 \
|
| 274 |
+
--data.transform.field.stat-path=assets/stats/seed_all_g1_motion39_textop_stats.json \
|
| 275 |
+
--data.transform.field.normalize-state \
|
| 276 |
+
--data.transform.field.pad-action-dim=39 \
|
| 277 |
+
--data.transform.field.pad-state-dim=39 \
|
| 278 |
+
--data.transform.model.resize.size 240 320 \
|
| 279 |
+
--data.transform.model.center_crop.size 240 320 \
|
| 280 |
+
--data.transform.model.no-img-aug \
|
| 281 |
+
--data.transform.model.text-only \
|
| 282 |
+
--data.transform.model.audio-only \
|
| 283 |
+
--data.transform.model.video-only \
|
| 284 |
+
--data.transform.model.video-size-pixels=$VIDEO_SIZE_PIXELS \
|
| 285 |
+
--model.model_name_or_path=$PSI_PRETRAIN_BACKBONE_PATH \
|
| 286 |
+
--model.noise-scheduler=flow \
|
| 287 |
+
--model.n_conditions=0 \
|
| 288 |
+
--model.action-chunk-size=$ACTION_CHUNK_SIZE \
|
| 289 |
+
--model.action-dim=39 \
|
| 290 |
+
--model.action-exec-horizon=$ACTION_EXEC_HORIZON \
|
| 291 |
+
--model.observation-horizon=$STATE_HORIZON \
|
| 292 |
+
--model.odim=39 \
|
| 293 |
+
--model.view_feature_dim=2048 \
|
| 294 |
+
--model.hidden-dim=$ACTION_HIDDEN_DIM \
|
| 295 |
+
--model.num-blocks=$ACTION_NUM_BLOCKS \
|
| 296 |
+
--model.nhead=$ACTION_NHEADS \
|
| 297 |
+
--model.use-future-progress-embedding \
|
| 298 |
+
--model.future-progress-dim=$PROGRESS_DIM \
|
| 299 |
+
--model.tune-vlm \
|
| 300 |
+
--model.lang-backbone-lr=1e-5 \
|
| 301 |
+
--model.no-use_film \
|
| 302 |
+
--model.no-combined_temb \
|
| 303 |
+
--model.rtc \
|
| 304 |
+
--model.max-delay=11 \
|
| 305 |
+
--train.max-grad-value=1 \
|
| 306 |
+
--train.loss-group-weight.joint-abs=1.0 \
|
| 307 |
+
--train.loss-group-weight.root-delta-xy=3.0 \
|
| 308 |
+
--train.loss-group-weight.root-rot5-delta=10.0 \
|
| 309 |
+
--train.loss-group-weight.root-height=15.0 \
|
| 310 |
+
--train.loss-group-weight.foot-contact=0.1
|
| 311 |
+
"
|
| 312 |
+
torchrun --nproc_per_node=$NPROC_PER_NODE --master_port=29500 scripts/train.py \
|
| 313 |
+
$args "$@"
|
posttrain-v1-psi0-all-7task-tune-vlm-direct-dense-actionhead-perfect-resume.sh
ADDED
|
@@ -0,0 +1,259 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#!/usr/bin/env bash
|
| 2 |
+
|
| 3 |
+
set -euo pipefail
|
| 4 |
+
|
| 5 |
+
ROOT=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
|
| 6 |
+
RUNS_BASE=$ROOT/runs/posttrain_v1_trimodal
|
| 7 |
+
TARGET=${1:-${PSI_RESUME_TARGET:-}}
|
| 8 |
+
|
| 9 |
+
cd "$ROOT"
|
| 10 |
+
|
| 11 |
+
source "$ROOT/scripts/activate_psi_env.sh"
|
| 12 |
+
[ -f .env ] && source .env
|
| 13 |
+
|
| 14 |
+
export PYTHONPATH="$ROOT/src${PYTHONPATH:+:$PYTHONPATH}"
|
| 15 |
+
export CUDA_VISIBLE_DEVICES=${CUDA_VISIBLE_DEVICES:-0,1,2,3,4,5,6,7}
|
| 16 |
+
export OMP_NUM_THREADS=${OMP_NUM_THREADS:-8}
|
| 17 |
+
export TF_CPP_MIN_LOG_LEVEL=${TF_CPP_MIN_LOG_LEVEL:-3}
|
| 18 |
+
export MASTER_PORT=${MASTER_PORT:-29500}
|
| 19 |
+
|
| 20 |
+
count_visible_devices() {
|
| 21 |
+
local devices=$1
|
| 22 |
+
local n=0
|
| 23 |
+
IFS=',' read -r -a parts <<< "$devices"
|
| 24 |
+
for part in "${parts[@]}"; do
|
| 25 |
+
if [ -n "$part" ]; then
|
| 26 |
+
n=$((n + 1))
|
| 27 |
+
fi
|
| 28 |
+
done
|
| 29 |
+
echo "$n"
|
| 30 |
+
}
|
| 31 |
+
|
| 32 |
+
mapfile -t RESUME_INFO < <(
|
| 33 |
+
python3 - "$RUNS_BASE" "$TARGET" <<'PY'
|
| 34 |
+
import json
|
| 35 |
+
import os
|
| 36 |
+
import re
|
| 37 |
+
import sys
|
| 38 |
+
|
| 39 |
+
runs_base = os.path.abspath(sys.argv[1])
|
| 40 |
+
target = sys.argv[2].strip()
|
| 41 |
+
|
| 42 |
+
|
| 43 |
+
def load_json(path):
|
| 44 |
+
with open(path, "r") as f:
|
| 45 |
+
return json.load(f)
|
| 46 |
+
|
| 47 |
+
|
| 48 |
+
def is_run_dir(path):
|
| 49 |
+
return (
|
| 50 |
+
os.path.isfile(os.path.join(path, "run_config.json"))
|
| 51 |
+
and os.path.isfile(os.path.join(path, "argv.txt"))
|
| 52 |
+
and os.path.isdir(os.path.join(path, "checkpoints"))
|
| 53 |
+
)
|
| 54 |
+
|
| 55 |
+
|
| 56 |
+
def pick_latest_run():
|
| 57 |
+
candidates = []
|
| 58 |
+
if not os.path.isdir(runs_base):
|
| 59 |
+
raise SystemExit(f"Run base does not exist: {runs_base}")
|
| 60 |
+
for name in os.listdir(runs_base):
|
| 61 |
+
run_dir = os.path.join(runs_base, name)
|
| 62 |
+
if not os.path.isdir(run_dir) or not is_run_dir(run_dir):
|
| 63 |
+
continue
|
| 64 |
+
cfg = load_json(os.path.join(run_dir, "run_config.json"))
|
| 65 |
+
timestamp = cfg.get("timestamp")
|
| 66 |
+
try:
|
| 67 |
+
ts_key = int(str(timestamp))
|
| 68 |
+
except Exception:
|
| 69 |
+
ts_key = -1
|
| 70 |
+
candidates.append((ts_key, os.path.getmtime(run_dir), run_dir))
|
| 71 |
+
if not candidates:
|
| 72 |
+
raise SystemExit(f"No resumable run found under {runs_base}")
|
| 73 |
+
candidates.sort()
|
| 74 |
+
return candidates[-1][2]
|
| 75 |
+
|
| 76 |
+
|
| 77 |
+
def best_checkpoint_for_run(run_dir):
|
| 78 |
+
checkpoint_root = os.path.join(run_dir, "checkpoints")
|
| 79 |
+
candidates = []
|
| 80 |
+
|
| 81 |
+
best_dir = os.path.join(checkpoint_root, "best_ckpt")
|
| 82 |
+
best_info = os.path.join(checkpoint_root, "best_eval_metric.json")
|
| 83 |
+
if os.path.isfile(os.path.join(best_dir, "model.safetensors")) and os.path.isfile(best_info):
|
| 84 |
+
info = load_json(best_info)
|
| 85 |
+
candidates.append((int(info["global_step"]), best_dir, "best_ckpt"))
|
| 86 |
+
|
| 87 |
+
for name in os.listdir(checkpoint_root):
|
| 88 |
+
m = re.fullmatch(r"ckpt_(\d+)", name)
|
| 89 |
+
if not m:
|
| 90 |
+
continue
|
| 91 |
+
ckpt_dir = os.path.join(checkpoint_root, name)
|
| 92 |
+
if os.path.isfile(os.path.join(ckpt_dir, "model.safetensors")):
|
| 93 |
+
candidates.append((int(m.group(1)), ckpt_dir, name))
|
| 94 |
+
|
| 95 |
+
if not candidates:
|
| 96 |
+
raise SystemExit(f"No usable checkpoint found under {checkpoint_root}")
|
| 97 |
+
|
| 98 |
+
candidates.sort(key=lambda x: (x[0], x[2]))
|
| 99 |
+
return candidates[-1]
|
| 100 |
+
|
| 101 |
+
|
| 102 |
+
def resolve_target():
|
| 103 |
+
if not target:
|
| 104 |
+
run_dir = pick_latest_run()
|
| 105 |
+
step, ckpt_dir, _ = best_checkpoint_for_run(run_dir)
|
| 106 |
+
return run_dir, ckpt_dir, step
|
| 107 |
+
|
| 108 |
+
path = os.path.abspath(target)
|
| 109 |
+
if is_run_dir(path):
|
| 110 |
+
step, ckpt_dir, _ = best_checkpoint_for_run(path)
|
| 111 |
+
return path, ckpt_dir, step
|
| 112 |
+
|
| 113 |
+
if os.path.isfile(os.path.join(path, "model.safetensors")):
|
| 114 |
+
run_dir = os.path.dirname(os.path.dirname(path))
|
| 115 |
+
if not is_run_dir(run_dir):
|
| 116 |
+
raise SystemExit(f"Checkpoint parent is not a valid run dir: {run_dir}")
|
| 117 |
+
|
| 118 |
+
name = os.path.basename(path)
|
| 119 |
+
if name == "best_ckpt":
|
| 120 |
+
info = load_json(os.path.join(os.path.dirname(path), "best_eval_metric.json"))
|
| 121 |
+
step = int(info["global_step"])
|
| 122 |
+
else:
|
| 123 |
+
m = re.fullmatch(r"ckpt_(\d+)", name)
|
| 124 |
+
if not m:
|
| 125 |
+
raise SystemExit(f"Unsupported checkpoint dir name: {name}")
|
| 126 |
+
step = int(m.group(1))
|
| 127 |
+
return run_dir, path, step
|
| 128 |
+
|
| 129 |
+
raise SystemExit(
|
| 130 |
+
"Target must be empty, a run dir, or an exact checkpoint dir with model.safetensors"
|
| 131 |
+
)
|
| 132 |
+
|
| 133 |
+
|
| 134 |
+
run_dir, ckpt_dir, resume_step = resolve_target()
|
| 135 |
+
cfg = load_json(os.path.join(run_dir, "run_config.json"))
|
| 136 |
+
wandb_cfg = cfg.get("wandb") or {}
|
| 137 |
+
env_cfg = cfg.get("environment_variables") or {}
|
| 138 |
+
|
| 139 |
+
expected_world_size = env_cfg.get("WORLD_SIZE") or ""
|
| 140 |
+
if not expected_world_size:
|
| 141 |
+
m = re.search(r"\.gpus(\d+)\.", str(wandb_cfg.get("name") or ""))
|
| 142 |
+
if m:
|
| 143 |
+
expected_world_size = m.group(1)
|
| 144 |
+
|
| 145 |
+
print(run_dir)
|
| 146 |
+
print(ckpt_dir)
|
| 147 |
+
print(str(resume_step))
|
| 148 |
+
print(str(wandb_cfg.get("id") or ""))
|
| 149 |
+
print(str(cfg.get("timestamp") or ""))
|
| 150 |
+
print(str(expected_world_size or ""))
|
| 151 |
+
PY
|
| 152 |
+
)
|
| 153 |
+
|
| 154 |
+
RUN_DIR=${RESUME_INFO[0]}
|
| 155 |
+
RESUME_CKPT_DIR=${RESUME_INFO[1]}
|
| 156 |
+
RESUME_STEP=${RESUME_INFO[2]}
|
| 157 |
+
WANDB_RUN_ID=${RESUME_INFO[3]}
|
| 158 |
+
RUN_TIMESTAMP=${RESUME_INFO[4]}
|
| 159 |
+
EXPECTED_WORLD_SIZE=${RESUME_INFO[5]}
|
| 160 |
+
|
| 161 |
+
if [ -z "$WANDB_RUN_ID" ]; then
|
| 162 |
+
echo "Missing wandb run id in $RUN_DIR/run_config.json"
|
| 163 |
+
exit 1
|
| 164 |
+
fi
|
| 165 |
+
|
| 166 |
+
NPROC_PER_NODE=$(count_visible_devices "$CUDA_VISIBLE_DEVICES")
|
| 167 |
+
if [ "$NPROC_PER_NODE" -le 0 ]; then
|
| 168 |
+
echo "Invalid CUDA_VISIBLE_DEVICES: $CUDA_VISIBLE_DEVICES"
|
| 169 |
+
exit 1
|
| 170 |
+
fi
|
| 171 |
+
|
| 172 |
+
if [ -n "$EXPECTED_WORLD_SIZE" ] && [ "$NPROC_PER_NODE" != "$EXPECTED_WORLD_SIZE" ]; then
|
| 173 |
+
echo "World size mismatch: original run used $EXPECTED_WORLD_SIZE GPUs, current CUDA_VISIBLE_DEVICES=$CUDA_VISIBLE_DEVICES gives $NPROC_PER_NODE"
|
| 174 |
+
exit 1
|
| 175 |
+
fi
|
| 176 |
+
|
| 177 |
+
ARGV_FILE=$RUN_DIR/argv.txt
|
| 178 |
+
if [ ! -f "$ARGV_FILE" ]; then
|
| 179 |
+
echo "Missing argv.txt: $ARGV_FILE"
|
| 180 |
+
exit 1
|
| 181 |
+
fi
|
| 182 |
+
|
| 183 |
+
ORIG_ARGS=()
|
| 184 |
+
while IFS= read -r line || [ -n "$line" ]; do
|
| 185 |
+
[ -z "$line" ] && continue
|
| 186 |
+
read -r -a parts <<< "$line"
|
| 187 |
+
ORIG_ARGS+=("${parts[@]}")
|
| 188 |
+
done < "$ARGV_FILE"
|
| 189 |
+
|
| 190 |
+
SKIP_KEYS=(
|
| 191 |
+
--train.resume_from_checkpoint
|
| 192 |
+
--train.resume_load_full_state
|
| 193 |
+
--train.skip_resumed_steps
|
| 194 |
+
--wandb.id
|
| 195 |
+
--wandb.resume
|
| 196 |
+
)
|
| 197 |
+
|
| 198 |
+
FILTERED_ARGS=()
|
| 199 |
+
i=0
|
| 200 |
+
while [ "$i" -lt "${#ORIG_ARGS[@]}" ]; do
|
| 201 |
+
token=${ORIG_ARGS[$i]}
|
| 202 |
+
matched=0
|
| 203 |
+
|
| 204 |
+
for key in "${SKIP_KEYS[@]}"; do
|
| 205 |
+
if [[ "$token" == "$key" ]]; then
|
| 206 |
+
matched=1
|
| 207 |
+
i=$((i + 1))
|
| 208 |
+
while [ "$i" -lt "${#ORIG_ARGS[@]}" ] && [[ ! "${ORIG_ARGS[$i]}" == --* ]]; do
|
| 209 |
+
i=$((i + 1))
|
| 210 |
+
done
|
| 211 |
+
break
|
| 212 |
+
fi
|
| 213 |
+
|
| 214 |
+
if [[ "$token" == "$key="* ]]; then
|
| 215 |
+
matched=1
|
| 216 |
+
i=$((i + 1))
|
| 217 |
+
break
|
| 218 |
+
fi
|
| 219 |
+
done
|
| 220 |
+
|
| 221 |
+
if [ "$matched" -eq 0 ]; then
|
| 222 |
+
FILTERED_ARGS+=("$token")
|
| 223 |
+
i=$((i + 1))
|
| 224 |
+
fi
|
| 225 |
+
done
|
| 226 |
+
|
| 227 |
+
FINAL_ARGS=(
|
| 228 |
+
"${FILTERED_ARGS[@]}"
|
| 229 |
+
"--train.resume_from_checkpoint=$RESUME_CKPT_DIR"
|
| 230 |
+
"--train.resume_load_full_state"
|
| 231 |
+
"--train.skip_resumed_steps"
|
| 232 |
+
"--wandb.id=$WANDB_RUN_ID"
|
| 233 |
+
"--wandb.resume=must"
|
| 234 |
+
)
|
| 235 |
+
|
| 236 |
+
export WANDB_RUN_ID
|
| 237 |
+
export WANDB_RESUME=must
|
| 238 |
+
|
| 239 |
+
echo "Perfect resume:"
|
| 240 |
+
echo " run_dir = $RUN_DIR"
|
| 241 |
+
echo " checkpoint = $RESUME_CKPT_DIR"
|
| 242 |
+
echo " resume_step = $RESUME_STEP"
|
| 243 |
+
echo " timestamp = $RUN_TIMESTAMP"
|
| 244 |
+
echo " wandb_id = $WANDB_RUN_ID"
|
| 245 |
+
echo " wandb_auth = configured"
|
| 246 |
+
echo " gpus = $CUDA_VISIBLE_DEVICES"
|
| 247 |
+
echo " nproc = $NPROC_PER_NODE"
|
| 248 |
+
echo " master_port = $MASTER_PORT"
|
| 249 |
+
|
| 250 |
+
if [ "${PSI_DRY_RUN:-0}" = "1" ]; then
|
| 251 |
+
echo " mode = dry-run"
|
| 252 |
+
exit 0
|
| 253 |
+
fi
|
| 254 |
+
|
| 255 |
+
exec torchrun \
|
| 256 |
+
--standalone \
|
| 257 |
+
--nproc_per_node="$NPROC_PER_NODE" \
|
| 258 |
+
--master_port="$MASTER_PORT" \
|
| 259 |
+
"${FINAL_ARGS[@]}"
|
posttrain-v1-psi0-all-7task-tune-vlm-direct-dense-full.sh
ADDED
|
@@ -0,0 +1,427 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
pkill -f monitor.py
|
| 2 |
+
|
| 3 |
+
PROJECT_ROOT=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
|
| 4 |
+
cd "$PROJECT_ROOT"
|
| 5 |
+
export PYTHONPATH="$PROJECT_ROOT/src${PYTHONPATH:+:$PYTHONPATH}"
|
| 6 |
+
export CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7
|
| 7 |
+
export PSI_OUTPUT_DIR=${PSI_OUTPUT_DIR:-$PROJECT_ROOT/runs}
|
| 8 |
+
export PSI_LOG_REPORT_TO=wandb
|
| 9 |
+
|
| 10 |
+
source "$PROJECT_ROOT/scripts/activate_psi_env.sh"
|
| 11 |
+
[ -f .env ] && source .env
|
| 12 |
+
|
| 13 |
+
|
| 14 |
+
export OMP_NUM_THREADS=8
|
| 15 |
+
export TF_CPP_MIN_LOG_LEVEL=3
|
| 16 |
+
|
| 17 |
+
# Keep JSON sources readable in this script, but pass compact JSON to tyro.
|
| 18 |
+
compact_json() {
|
| 19 |
+
python -c 'import json, sys; print(json.dumps(json.load(sys.stdin), separators=(",", ":")))'
|
| 20 |
+
}
|
| 21 |
+
|
| 22 |
+
PSI_SEED_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/Psi0/dataset/seed_all
|
| 23 |
+
PSI_SEED_METADATA=metadata/seed_metadata_v004.csv
|
| 24 |
+
PSI_TEXT_RECORDS_CACHE_DIR=${PSI_TEXT_RECORDS_CACHE_DIR:-$PSI_SEED_ROOT/.index_cache_multimodal_yth}
|
| 25 |
+
|
| 26 |
+
PSI_DANCE_MOTION_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/GR00T-WholeBodyControl/gear_sonic_deploy/reference/converted/psi0_dance/gvhmr
|
| 27 |
+
PSI_DANCE_AUDIO_VIDEO_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/dance_1723_video_psi0/dance_1723_video_10s
|
| 28 |
+
PSI_DANCE_VIDEO_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/dance_1723_video_psi0/dance_1723_video_10s
|
| 29 |
+
PSI_DANCE_VIDEO_EMBED_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/dance_1723_video_psi0/dance_1723_video_10s_embeds
|
| 30 |
+
PSI_VIDEO_DEMO_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/videogen/motion_example_0506_demo
|
| 31 |
+
PSI_2264_WITH_AUDIO_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/videogen/2264_with_audio
|
| 32 |
+
PSI_QWEN3_TTS_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/qwen3-tts
|
| 33 |
+
PSI_VIDEO_CLIP_DATASET_ROOT=${PSI_VIDEO_CLIP_DATASET_ROOT:-/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/videogen/psi0_video_clip_50f_dense_stride1_direct_full}
|
| 34 |
+
PSI_DANCE_VIDEO_CLIP_ROOT=$PSI_VIDEO_CLIP_DATASET_ROOT/dance_1723_video_10s
|
| 35 |
+
PSI_MOTION_EXAMPLE_VIDEO_CLIP_ROOT=$PSI_VIDEO_CLIP_DATASET_ROOT/motion_example_0506_demo
|
| 36 |
+
PSI_2264_WITH_AUDIO_VIDEO_CLIP_ROOT=$PSI_VIDEO_CLIP_DATASET_ROOT/2264_with_audio
|
| 37 |
+
PSI_DANCE_VIDEO_CLIP_EMBED_ROOT=${PSI_DANCE_VIDEO_CLIP_EMBED_ROOT:-$PSI_DANCE_VIDEO_CLIP_ROOT/embeds}
|
| 38 |
+
PSI_MOTION_EXAMPLE_VIDEO_CLIP_EMBED_ROOT=${PSI_MOTION_EXAMPLE_VIDEO_CLIP_EMBED_ROOT:-$PSI_MOTION_EXAMPLE_VIDEO_CLIP_ROOT/embeds}
|
| 39 |
+
PSI_2264_WITH_AUDIO_VIDEO_CLIP_EMBED_ROOT=${PSI_2264_WITH_AUDIO_VIDEO_CLIP_EMBED_ROOT:-$PSI_2264_WITH_AUDIO_VIDEO_CLIP_ROOT/embeds}
|
| 40 |
+
PSI_MOTION_EXAMPLE_VIDEO_EMBED_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/videogen/motion_example_0506_demo_video_embeds
|
| 41 |
+
PSI_AUDIO_RECORDS_CACHE_DIR=${PSI_AUDIO_RECORDS_CACHE_DIR:-$PSI_2264_WITH_AUDIO_ROOT/.index_cache_multimodal_yth}
|
| 42 |
+
PSI_AUDIO_COMMAND_RECORDS_CACHE_DIR=${PSI_AUDIO_COMMAND_RECORDS_CACHE_DIR:-$PSI_QWEN3_TTS_ROOT/.index_cache_multimodal_yth}
|
| 43 |
+
PSI_DANCE_AUDIO_NPY_ROOT=${PSI_DANCE_AUDIO_NPY_ROOT:-$PSI_DANCE_AUDIO_VIDEO_ROOT/audio_npy_yth}
|
| 44 |
+
PSI_DANCE_AUDIO_NPY_MANIFEST=${PSI_DANCE_AUDIO_NPY_MANIFEST:-$PSI_DANCE_AUDIO_VIDEO_ROOT/metadata_audio_npy.jsonl}
|
| 45 |
+
PSI_2264_AUDIO_NPY_ROOT=${PSI_2264_AUDIO_NPY_ROOT:-$PSI_2264_WITH_AUDIO_ROOT/audio_npy_yth}
|
| 46 |
+
PSI_2264_AUDIO_NPY_MANIFEST=${PSI_2264_AUDIO_NPY_MANIFEST:-$PSI_2264_WITH_AUDIO_ROOT/metadata_audio_npy.jsonl}
|
| 47 |
+
PSI_QWEN3_TTS_AUDIO_NPY_ROOT=${PSI_QWEN3_TTS_AUDIO_NPY_ROOT:-$PSI_QWEN3_TTS_ROOT/audio_npy_yth}
|
| 48 |
+
PSI_QWEN3_TTS_AUDIO_NPY_MANIFEST=${PSI_QWEN3_TTS_AUDIO_NPY_MANIFEST:-$PSI_QWEN3_TTS_ROOT/metadata_audio_npy.jsonl}
|
| 49 |
+
|
| 50 |
+
if [ -f "$PSI_2264_AUDIO_NPY_MANIFEST" ]; then
|
| 51 |
+
PSI_2264_AUDIO_METADATA_JSONL=$PSI_2264_AUDIO_NPY_MANIFEST
|
| 52 |
+
PSI_2264_AUDIO_PATH_KEY=audio_path
|
| 53 |
+
else
|
| 54 |
+
PSI_2264_AUDIO_METADATA_JSONL=metadata_convert.jsonl
|
| 55 |
+
PSI_2264_AUDIO_PATH_KEY=video_path
|
| 56 |
+
fi
|
| 57 |
+
|
| 58 |
+
if [ -f "$PSI_DANCE_AUDIO_NPY_MANIFEST" ]; then
|
| 59 |
+
PSI_DANCE_AUDIO_SOURCE_JSON=$(cat <<JSON
|
| 60 |
+
{
|
| 61 |
+
"name": "psi0_dance_audio",
|
| 62 |
+
"dataset_root": "$PSI_DANCE_AUDIO_VIDEO_ROOT",
|
| 63 |
+
"metadata_jsonl": "$PSI_DANCE_AUDIO_NPY_MANIFEST",
|
| 64 |
+
"motion_dir_key": "converted_path",
|
| 65 |
+
"audio_path_key": "audio_path",
|
| 66 |
+
"records_cache_dir": "$PSI_AUDIO_RECORDS_CACHE_DIR",
|
| 67 |
+
"dataset_name": "seed-audio-g1"
|
| 68 |
+
}
|
| 69 |
+
JSON
|
| 70 |
+
)
|
| 71 |
+
else
|
| 72 |
+
PSI_DANCE_AUDIO_SOURCE_JSON=$(cat <<JSON
|
| 73 |
+
{
|
| 74 |
+
"name": "psi0_dance_audio",
|
| 75 |
+
"root_dir": "$PSI_DANCE_MOTION_ROOT",
|
| 76 |
+
"video_dir": "$PSI_DANCE_AUDIO_VIDEO_ROOT",
|
| 77 |
+
"records_cache_dir": "$PSI_AUDIO_RECORDS_CACHE_DIR",
|
| 78 |
+
"dataset_name": "seed-audio-g1"
|
| 79 |
+
}
|
| 80 |
+
JSON
|
| 81 |
+
)
|
| 82 |
+
fi
|
| 83 |
+
|
| 84 |
+
if [ -z "$PSI_AUDIO_SOURCES_JSON" ]; then
|
| 85 |
+
PSI_AUDIO_SOURCES_JSON=$(compact_json <<JSON
|
| 86 |
+
[
|
| 87 |
+
$PSI_DANCE_AUDIO_SOURCE_JSON,
|
| 88 |
+
{
|
| 89 |
+
"name": "2264_with_audio",
|
| 90 |
+
"dataset_root": "$PSI_2264_WITH_AUDIO_ROOT",
|
| 91 |
+
"metadata_jsonl": "$PSI_2264_AUDIO_METADATA_JSONL",
|
| 92 |
+
"motion_dir_key": "converted_path",
|
| 93 |
+
"audio_path_key": "$PSI_2264_AUDIO_PATH_KEY",
|
| 94 |
+
"records_cache_dir": "$PSI_AUDIO_RECORDS_CACHE_DIR",
|
| 95 |
+
"dataset_name": "2264-with-audio-g1"
|
| 96 |
+
}
|
| 97 |
+
]
|
| 98 |
+
JSON
|
| 99 |
+
)
|
| 100 |
+
fi
|
| 101 |
+
|
| 102 |
+
if [ -f "$PSI_QWEN3_TTS_AUDIO_NPY_MANIFEST" ]; then
|
| 103 |
+
PSI_AUDIO_COMMAND_SOURCE_JSON=$(cat <<JSON
|
| 104 |
+
{
|
| 105 |
+
"name": "qwen3_tts_seed_audio_command",
|
| 106 |
+
"root_dir": "$PSI_SEED_ROOT",
|
| 107 |
+
"dataset_root": "$PSI_QWEN3_TTS_ROOT",
|
| 108 |
+
"metadata_jsonl": "$PSI_QWEN3_TTS_AUDIO_NPY_MANIFEST",
|
| 109 |
+
"motion_dir_key": "filename",
|
| 110 |
+
"audio_path_key": "audio_path",
|
| 111 |
+
"instruction_key": "content_short_description",
|
| 112 |
+
"motion_metadata_csv": "$PSI_SEED_METADATA",
|
| 113 |
+
"motion_metadata_key": "filename",
|
| 114 |
+
"motion_metadata_path_key": "move_g1_path",
|
| 115 |
+
"audio_file_extensions": ".npy",
|
| 116 |
+
"records_cache_dir": "$PSI_AUDIO_COMMAND_RECORDS_CACHE_DIR",
|
| 117 |
+
"audio_task": "command",
|
| 118 |
+
"dataset_name": "seed-audio-command-g1"
|
| 119 |
+
}
|
| 120 |
+
JSON
|
| 121 |
+
)
|
| 122 |
+
else
|
| 123 |
+
PSI_AUDIO_COMMAND_SOURCE_JSON=$(cat <<JSON
|
| 124 |
+
{
|
| 125 |
+
"name": "qwen3_tts_seed_audio_command",
|
| 126 |
+
"root_dir": "$PSI_SEED_ROOT",
|
| 127 |
+
"dataset_root": "$PSI_QWEN3_TTS_ROOT",
|
| 128 |
+
"metadata_jsonl": "metadata.jsonl",
|
| 129 |
+
"motion_dir_key": "filename",
|
| 130 |
+
"audio_path_key": "audio_path",
|
| 131 |
+
"instruction_key": "content_short_description",
|
| 132 |
+
"motion_metadata_csv": "$PSI_SEED_METADATA",
|
| 133 |
+
"motion_metadata_key": "filename",
|
| 134 |
+
"motion_metadata_path_key": "move_g1_path",
|
| 135 |
+
"audio_file_extensions": ".wav",
|
| 136 |
+
"records_cache_dir": "$PSI_AUDIO_COMMAND_RECORDS_CACHE_DIR",
|
| 137 |
+
"audio_task": "command",
|
| 138 |
+
"dataset_name": "seed-audio-command-g1"
|
| 139 |
+
}
|
| 140 |
+
JSON
|
| 141 |
+
)
|
| 142 |
+
fi
|
| 143 |
+
|
| 144 |
+
if [ -z "$PSI_AUDIO_COMMAND_SOURCES_JSON" ]; then
|
| 145 |
+
PSI_AUDIO_COMMAND_SOURCES_JSON=$(compact_json <<JSON
|
| 146 |
+
[
|
| 147 |
+
$PSI_AUDIO_COMMAND_SOURCE_JSON
|
| 148 |
+
]
|
| 149 |
+
JSON
|
| 150 |
+
)
|
| 151 |
+
fi
|
| 152 |
+
|
| 153 |
+
if [ -z "$PSI_VIDEO_SOURCES_JSON" ]; then
|
| 154 |
+
PSI_VIDEO_SOURCES_JSON=$(compact_json <<JSON
|
| 155 |
+
[
|
| 156 |
+
{
|
| 157 |
+
"name": "psi0_dance_gvhmr",
|
| 158 |
+
"root_dir": "$PSI_DANCE_MOTION_ROOT",
|
| 159 |
+
"video_dir": "$PSI_DANCE_VIDEO_ROOT",
|
| 160 |
+
"video_embed_dir": "$PSI_DANCE_VIDEO_EMBED_ROOT",
|
| 161 |
+
"dataset_name": "seed-video-g1"
|
| 162 |
+
},
|
| 163 |
+
{
|
| 164 |
+
"name": "motion_example_0506_demo",
|
| 165 |
+
"dataset_root": "$PSI_VIDEO_DEMO_ROOT",
|
| 166 |
+
"metadata_jsonl": "metadata.jsonl",
|
| 167 |
+
"motion_dir_key": "converted_path",
|
| 168 |
+
"video_path_key": "video_path",
|
| 169 |
+
"instruction_key": "video_caption",
|
| 170 |
+
"video_embed_dir": "$PSI_MOTION_EXAMPLE_VIDEO_EMBED_ROOT",
|
| 171 |
+
"dataset_name": "motion-example-video-g1"
|
| 172 |
+
}
|
| 173 |
+
]
|
| 174 |
+
JSON
|
| 175 |
+
)
|
| 176 |
+
fi
|
| 177 |
+
|
| 178 |
+
if [ -z "$PSI_VIDEO_CLIP_SOURCES_JSON" ]; then
|
| 179 |
+
PSI_VIDEO_CLIP_SOURCES_JSON=$(compact_json <<JSON
|
| 180 |
+
[
|
| 181 |
+
{
|
| 182 |
+
"name": "dance_1723_video_10s_clip50",
|
| 183 |
+
"dataset_root": "$PSI_DANCE_VIDEO_CLIP_ROOT",
|
| 184 |
+
"metadata_jsonl": "metadata.jsonl",
|
| 185 |
+
"motion_dir_key": "converted_path",
|
| 186 |
+
"video_path_key": "video_path",
|
| 187 |
+
"instruction_key": "video_caption",
|
| 188 |
+
"video_embed_dir": "$PSI_DANCE_VIDEO_CLIP_EMBED_ROOT",
|
| 189 |
+
"dataset_name": "dance-video-clip-g1"
|
| 190 |
+
},
|
| 191 |
+
{
|
| 192 |
+
"name": "motion_example_0506_demo_clip50",
|
| 193 |
+
"dataset_root": "$PSI_MOTION_EXAMPLE_VIDEO_CLIP_ROOT",
|
| 194 |
+
"metadata_jsonl": "metadata.jsonl",
|
| 195 |
+
"motion_dir_key": "converted_path",
|
| 196 |
+
"video_path_key": "video_path",
|
| 197 |
+
"instruction_key": "video_caption",
|
| 198 |
+
"video_embed_dir": "$PSI_MOTION_EXAMPLE_VIDEO_CLIP_EMBED_ROOT",
|
| 199 |
+
"dataset_name": "motion-example-video-clip-g1"
|
| 200 |
+
},
|
| 201 |
+
{
|
| 202 |
+
"name": "2264_with_audio_clip50",
|
| 203 |
+
"dataset_root": "$PSI_2264_WITH_AUDIO_VIDEO_CLIP_ROOT",
|
| 204 |
+
"metadata_jsonl": "metadata.jsonl",
|
| 205 |
+
"motion_dir_key": "converted_path",
|
| 206 |
+
"video_path_key": "video_path",
|
| 207 |
+
"instruction_key": "video_caption",
|
| 208 |
+
"video_embed_dir": "$PSI_2264_WITH_AUDIO_VIDEO_CLIP_EMBED_ROOT",
|
| 209 |
+
"dataset_name": "2264-with-audio-video-clip-g1"
|
| 210 |
+
}
|
| 211 |
+
]
|
| 212 |
+
JSON
|
| 213 |
+
)
|
| 214 |
+
fi
|
| 215 |
+
|
| 216 |
+
# 7-task from-scratch mix, excluding legacy VIDEO_TEXT / VIDEO_ONLY.
|
| 217 |
+
# Smaller data sources get larger weights. These use a tempered inverse-size
|
| 218 |
+
# rule, (47M / source_size)^0.25, so rare modalities are upweighted without
|
| 219 |
+
# completely starving text/video_clip in BatchMixtureSampler.
|
| 220 |
+
# text stride1 frame samples: ~47,000,000
|
| 221 |
+
# music audio dance frame samples: 213,731
|
| 222 |
+
# audio_command wav files: 41,760
|
| 223 |
+
# dense video_clip metadata: 4,597,576
|
| 224 |
+
PSI_TEXT_TASK_WEIGHT=1
|
| 225 |
+
PSI_AUDIO_TEXT_TASK_WEIGHT=1
|
| 226 |
+
PSI_AUDIO_ONLY_TASK_WEIGHT=1
|
| 227 |
+
PSI_AUDIO_COMMAND_TEXT_TASK_WEIGHT=1
|
| 228 |
+
PSI_AUDIO_COMMAND_ONLY_TASK_WEIGHT=1
|
| 229 |
+
PSI_VIDEO_TEXT_TASK_WEIGHT=0
|
| 230 |
+
PSI_VIDEO_ONLY_TASK_WEIGHT=0
|
| 231 |
+
PSI_VIDEO_CLIP_TEXT_TASK_WEIGHT=1
|
| 232 |
+
PSI_VIDEO_CLIP_ONLY_TASK_WEIGHT=1
|
| 233 |
+
|
| 234 |
+
PSI_PRETRAIN_BACKBONE_PATH=/cpfs01/VideoGen/public_ckpt/Qwen/Qwen2.5-Omni-3B
|
| 235 |
+
NUM_PAST_FRAMES=0
|
| 236 |
+
STATE_HORIZON=$((NUM_PAST_FRAMES + 1))
|
| 237 |
+
ACTION_CHUNK_SIZE=30
|
| 238 |
+
ACTION_EXEC_HORIZON=20
|
| 239 |
+
RTC_PREFIX_FRAMES=$((ACTION_CHUNK_SIZE - ACTION_EXEC_HORIZON))
|
| 240 |
+
PROGRESS_DIM=5
|
| 241 |
+
MOTION_FPS=50
|
| 242 |
+
AUDIO_WINDOW_SECONDS=$(awk "BEGIN {printf \"%.6f\", ${ACTION_CHUNK_SIZE} / ${MOTION_FPS}}")
|
| 243 |
+
AUDIO_SAMPLE_STEP_FRAMES=$ACTION_EXEC_HORIZON
|
| 244 |
+
VIDEO_SAMPLE_FPS=1.0
|
| 245 |
+
VIDEO_CLIP_SAMPLE_FPS=8.0
|
| 246 |
+
VIDEO_SIZE_PIXELS=50176
|
| 247 |
+
VIDEO_DECODER=ffmpeg
|
| 248 |
+
VIDEO_CACHE_SIZE=64
|
| 249 |
+
VIDEO_CLIP_MOTION_CACHE_SIZE=256
|
| 250 |
+
VIDEO_RECORD_BUILD_WORKERS=${VIDEO_RECORD_BUILD_WORKERS:-32}
|
| 251 |
+
VIDEO_RECORD_BUILD_CHUNK_SIZE=${VIDEO_RECORD_BUILD_CHUNK_SIZE:-4096}
|
| 252 |
+
VIDEO_CLIP_TRUST_METADATA=${VIDEO_CLIP_TRUST_METADATA:-1}
|
| 253 |
+
VIDEO_CLIP_RECORDS_CACHE_DIR=${VIDEO_CLIP_RECORDS_CACHE_DIR:-$PSI_VIDEO_CLIP_DATASET_ROOT/.index_cache_multimodal_yth}
|
| 254 |
+
if [ "$VIDEO_CLIP_TRUST_METADATA" = "1" ] || [ "$VIDEO_CLIP_TRUST_METADATA" = "true" ] || [ "$VIDEO_CLIP_TRUST_METADATA" = "True" ]; then
|
| 255 |
+
VIDEO_CLIP_TRUST_METADATA_FLAG="--data.video-clip.trust-metadata"
|
| 256 |
+
else
|
| 257 |
+
VIDEO_CLIP_TRUST_METADATA_FLAG="--data.video-clip.no-trust-metadata"
|
| 258 |
+
fi
|
| 259 |
+
|
| 260 |
+
# DATALOADER_NUM_WORKERS=4
|
| 261 |
+
# VAL_DATALOADER_NUM_WORKERS=2
|
| 262 |
+
|
| 263 |
+
# DATALOADER_NUM_WORKERS=${DATALOADER_NUM_WORKERS:-0}
|
| 264 |
+
# VAL_DATALOADER_NUM_WORKERS=${VAL_DATALOADER_NUM_WORKERS:-0}
|
| 265 |
+
|
| 266 |
+
DATALOADER_NUM_WORKERS=${DATALOADER_NUM_WORKERS:-2}
|
| 267 |
+
VAL_DATALOADER_NUM_WORKERS=${VAL_DATALOADER_NUM_WORKERS:-2}
|
| 268 |
+
|
| 269 |
+
NPROC_PER_NODE=$(echo $CUDA_VISIBLE_DEVICES | tr ',' '\n' | wc -l)
|
| 270 |
+
|
| 271 |
+
echo "Training with $NPROC_PER_NODE GPU(s): $CUDA_VISIBLE_DEVICES"
|
| 272 |
+
echo "Text root: $PSI_SEED_ROOT"
|
| 273 |
+
echo "Text records cache dir: $PSI_TEXT_RECORDS_CACHE_DIR"
|
| 274 |
+
echo "Dance motion root: $PSI_DANCE_MOTION_ROOT"
|
| 275 |
+
echo "Dance audio/video root: $PSI_DANCE_AUDIO_VIDEO_ROOT"
|
| 276 |
+
echo "Dance video root: $PSI_DANCE_VIDEO_ROOT"
|
| 277 |
+
echo "Dance video embed root: $PSI_DANCE_VIDEO_EMBED_ROOT"
|
| 278 |
+
echo "2264 with audio root: $PSI_2264_WITH_AUDIO_ROOT"
|
| 279 |
+
echo "Dance audio npy root: $PSI_DANCE_AUDIO_NPY_ROOT"
|
| 280 |
+
echo "Dance audio manifest: $PSI_DANCE_AUDIO_NPY_MANIFEST"
|
| 281 |
+
echo "2264 audio npy root: $PSI_2264_AUDIO_NPY_ROOT"
|
| 282 |
+
echo "2264 audio manifest: $PSI_2264_AUDIO_METADATA_JSONL"
|
| 283 |
+
echo "Audio records cache dir: $PSI_AUDIO_RECORDS_CACHE_DIR"
|
| 284 |
+
echo "Audio command records cache dir: $PSI_AUDIO_COMMAND_RECORDS_CACHE_DIR"
|
| 285 |
+
echo "Qwen3-TTS acdudio command root: $PSI_QWEN3_TTS_ROOT"
|
| 286 |
+
echo "Qwen3-TTS audio npy root: $PSI_QWEN3_TTS_AUDIO_NPY_ROOT"
|
| 287 |
+
echo "Qwen3-TTS audio manifest: $PSI_QWEN3_TTS_AUDIO_NPY_MANIFEST"
|
| 288 |
+
echo "Audio sources: $PSI_AUDIO_SOURCES_JSON"
|
| 289 |
+
echo "Audio command sources: $PSI_AUDIO_COMMAND_SOURCES_JSON"
|
| 290 |
+
echo "Video sources: $PSI_VIDEO_SOURCES_JSON"
|
| 291 |
+
echo "Video clip sources: $PSI_VIDEO_CLIP_SOURCES_JSON"
|
| 292 |
+
echo "DataLoader workers: train=$DATALOADER_NUM_WORKERS val=$VAL_DATALOADER_NUM_WORKERS"
|
| 293 |
+
echo "Sampler: task_mixture"
|
| 294 |
+
echo "Task weights: text=$PSI_TEXT_TASK_WEIGHT audio_text=$PSI_AUDIO_TEXT_TASK_WEIGHT audio_only=$PSI_AUDIO_ONLY_TASK_WEIGHT audio_command_text=$PSI_AUDIO_COMMAND_TEXT_TASK_WEIGHT audio_command_only=$PSI_AUDIO_COMMAND_ONLY_TASK_WEIGHT video_text=$PSI_VIDEO_TEXT_TASK_WEIGHT video_only=$PSI_VIDEO_ONLY_TASK_WEIGHT video_clip_text=$PSI_VIDEO_CLIP_TEXT_TASK_WEIGHT video_clip_only=$PSI_VIDEO_CLIP_ONLY_TASK_WEIGHT"
|
| 295 |
+
# echo "Resume run dir: $PSI_PRETRAIN_RUN_DIR"
|
| 296 |
+
echo "TextOp 39D action/state enabled"
|
| 297 |
+
echo "Future progress embedding: enabled, dim $PROGRESS_DIM"
|
| 298 |
+
echo "Backbone path: $PSI_PRETRAIN_BACKBONE_PATH"
|
| 299 |
+
echo "Output dir: $PSI_OUTPUT_DIR"
|
| 300 |
+
echo "Aligned audio window seconds: $AUDIO_WINDOW_SECONDS"
|
| 301 |
+
echo "Aligned audio sample step frames: $AUDIO_SAMPLE_STEP_FRAMES"
|
| 302 |
+
echo "Video clip trust metadata: $VIDEO_CLIP_TRUST_METADATA"
|
| 303 |
+
echo "Video clip records cache dir: $VIDEO_CLIP_RECORDS_CACHE_DIR"
|
| 304 |
+
echo "Video record build workers: $VIDEO_RECORD_BUILD_WORKERS"
|
| 305 |
+
echo "Video record build chunk size: $VIDEO_RECORD_BUILD_CHUNK_SIZE"
|
| 306 |
+
|
| 307 |
+
args="posttrain_v1_seed_text_audio_video_mix_psi0_config \
|
| 308 |
+
--seed=292285 \
|
| 309 |
+
--exp=posttrain \
|
| 310 |
+
--timestamp=$(date +"%y%m%d%H%M") \
|
| 311 |
+
--train.name=posttrain_v1_trimodal \
|
| 312 |
+
--train.data_parallel=ddp \
|
| 313 |
+
--train.mixed_precision=bf16 \
|
| 314 |
+
--train.train_batch_size=64 \
|
| 315 |
+
--train.val_batch_size=64 \
|
| 316 |
+
--train.num-workers=$DATALOADER_NUM_WORKERS \
|
| 317 |
+
--train.val-num-workers=$VAL_DATALOADER_NUM_WORKERS \
|
| 318 |
+
--train.resume_from_checkpoint=None \
|
| 319 |
+
--train.max_checkpoints_to_keep=1 \
|
| 320 |
+
--train.gradient_accumulation_steps=1 \
|
| 321 |
+
--train.learning_rate=1e-4 \
|
| 322 |
+
--train.max_training_steps=500000 \
|
| 323 |
+
--train.warmup_ratio=None \
|
| 324 |
+
--train.warmup_steps=1000 \
|
| 325 |
+
--train.checkpointing_steps=1000 \
|
| 326 |
+
--train.validation_steps=1000 \
|
| 327 |
+
--train.val_num_batches=20 \
|
| 328 |
+
--train.lr_scheduler_type=cosine_with_min_lr \
|
| 329 |
+
--train.lr_scheduler_kwargs.betas 0.9 0.999 \
|
| 330 |
+
--train.lr_scheduler_kwargs.weight_decay=0 \
|
| 331 |
+
--train.lr_scheduler_kwargs.eps=1e-8 \
|
| 332 |
+
--log.report_to=$PSI_LOG_REPORT_TO \
|
| 333 |
+
--train.output_dir=$PSI_OUTPUT_DIR \
|
| 334 |
+
--data.text.root-dir=$PSI_SEED_ROOT \
|
| 335 |
+
--data.text.metadata-csv=$PSI_SEED_METADATA \
|
| 336 |
+
--data.text.records-cache-dir=$PSI_TEXT_RECORDS_CACHE_DIR \
|
| 337 |
+
--data.text-ratio=$PSI_TEXT_TASK_WEIGHT \
|
| 338 |
+
--data.audio.sources-json=$PSI_AUDIO_SOURCES_JSON \
|
| 339 |
+
--data.audio-ratio=$PSI_AUDIO_TEXT_TASK_WEIGHT \
|
| 340 |
+
--data.audio-only-ratio=$PSI_AUDIO_ONLY_TASK_WEIGHT \
|
| 341 |
+
--data.audio.sample-stride=1 \
|
| 342 |
+
--data.audio.sample-step-frames=$AUDIO_SAMPLE_STEP_FRAMES \
|
| 343 |
+
--data.audio.motion-fps=$MOTION_FPS \
|
| 344 |
+
--data.audio.audio-sampling-rate=16000 \
|
| 345 |
+
--data.audio.audio-window-seconds=$AUDIO_WINDOW_SECONDS \
|
| 346 |
+
--data.audio.audio-left-context-seconds=0.0 \
|
| 347 |
+
--data.audio-command.sources-json=$PSI_AUDIO_COMMAND_SOURCES_JSON \
|
| 348 |
+
--data.audio-command-ratio=$PSI_AUDIO_COMMAND_TEXT_TASK_WEIGHT \
|
| 349 |
+
--data.audio-command-only-ratio=$PSI_AUDIO_COMMAND_ONLY_TASK_WEIGHT \
|
| 350 |
+
--data.audio-command.sample-stride=1 \
|
| 351 |
+
--data.audio-command.sample-step-frames=$AUDIO_SAMPLE_STEP_FRAMES \
|
| 352 |
+
--data.audio-command.motion-fps=$MOTION_FPS \
|
| 353 |
+
--data.audio-command.audio-sampling-rate=16000 \
|
| 354 |
+
--data.audio-command.audio-window-seconds=$AUDIO_WINDOW_SECONDS \
|
| 355 |
+
--data.audio-command.audio-left-context-seconds=0.0 \
|
| 356 |
+
--data.audio-command.full-audio \
|
| 357 |
+
--data.video-ratio=$PSI_VIDEO_TEXT_TASK_WEIGHT \
|
| 358 |
+
--data.video-only-ratio=$PSI_VIDEO_ONLY_TASK_WEIGHT \
|
| 359 |
+
--data.video.sample-stride=1 \
|
| 360 |
+
--data.video.motion-fps=$MOTION_FPS \
|
| 361 |
+
--data.video.video-sample-fps=$VIDEO_SAMPLE_FPS \
|
| 362 |
+
--data.video.video-size-pixels=$VIDEO_SIZE_PIXELS \
|
| 363 |
+
--data.video.video-decoder=$VIDEO_DECODER \
|
| 364 |
+
--data.video.video-cache-size=$VIDEO_CACHE_SIZE \
|
| 365 |
+
--data.video.record-build-workers=$VIDEO_RECORD_BUILD_WORKERS \
|
| 366 |
+
--data.video.record-build-chunk-size=$VIDEO_RECORD_BUILD_CHUNK_SIZE \
|
| 367 |
+
--data.video.sources-json=$PSI_VIDEO_SOURCES_JSON \
|
| 368 |
+
--data.video-clip-text-ratio=$PSI_VIDEO_CLIP_TEXT_TASK_WEIGHT \
|
| 369 |
+
--data.video-clip-only-ratio=$PSI_VIDEO_CLIP_ONLY_TASK_WEIGHT \
|
| 370 |
+
--data.video-clip.sources-json=$PSI_VIDEO_CLIP_SOURCES_JSON \
|
| 371 |
+
--data.video-clip.sample-stride=1 \
|
| 372 |
+
--data.video-clip.motion-fps=$MOTION_FPS \
|
| 373 |
+
--data.video-clip.video-sample-fps=$VIDEO_CLIP_SAMPLE_FPS \
|
| 374 |
+
--data.video-clip.video-size-pixels=$VIDEO_SIZE_PIXELS \
|
| 375 |
+
--data.video-clip.video-decoder=$VIDEO_DECODER \
|
| 376 |
+
--data.video-clip.video-cache-size=$VIDEO_CACHE_SIZE \
|
| 377 |
+
--data.video-clip.motion-cache-size=$VIDEO_CLIP_MOTION_CACHE_SIZE \
|
| 378 |
+
--data.video-clip.record-build-workers=$VIDEO_RECORD_BUILD_WORKERS \
|
| 379 |
+
--data.video-clip.record-build-chunk-size=$VIDEO_RECORD_BUILD_CHUNK_SIZE \
|
| 380 |
+
$VIDEO_CLIP_TRUST_METADATA_FLAG \
|
| 381 |
+
--data.video-clip.records-cache-dir=$VIDEO_CLIP_RECORDS_CACHE_DIR \
|
| 382 |
+
--data.video-clip.action-exec-horizon=$ACTION_EXEC_HORIZON \
|
| 383 |
+
--data.sampler=task_mixture \
|
| 384 |
+
--data.val-task=text,audio,audio_only,audio_command,audio_command_only,video_clip,video_clip_only \
|
| 385 |
+
--data.transform.repack.num-past-frames=$NUM_PAST_FRAMES \
|
| 386 |
+
--data.transform.repack.textop-actions \
|
| 387 |
+
--data.transform.repack.action-chunk-size=$ACTION_CHUNK_SIZE \
|
| 388 |
+
--data.transform.repack.rtc-prefix-frames=$RTC_PREFIX_FRAMES \
|
| 389 |
+
--data.transform.repack.pad-action-dim=39 \
|
| 390 |
+
--data.transform.repack.pad-state-dim=39 \
|
| 391 |
+
--data.transform.field.stat-path=assets/stats/seed_all_g1_motion39_textop_stats.json \
|
| 392 |
+
--data.transform.field.normalize-state \
|
| 393 |
+
--data.transform.field.pad-action-dim=39 \
|
| 394 |
+
--data.transform.field.pad-state-dim=39 \
|
| 395 |
+
--data.transform.model.resize.size 240 320 \
|
| 396 |
+
--data.transform.model.center_crop.size 240 320 \
|
| 397 |
+
--data.transform.model.no-img-aug \
|
| 398 |
+
--data.transform.model.text-only \
|
| 399 |
+
--data.transform.model.audio-only \
|
| 400 |
+
--data.transform.model.video-only \
|
| 401 |
+
--data.transform.model.video-size-pixels=$VIDEO_SIZE_PIXELS \
|
| 402 |
+
--model.model_name_or_path=$PSI_PRETRAIN_BACKBONE_PATH \
|
| 403 |
+
--model.noise-scheduler=flow \
|
| 404 |
+
--model.n_conditions=0 \
|
| 405 |
+
--model.action-chunk-size=$ACTION_CHUNK_SIZE \
|
| 406 |
+
--model.action-dim=39 \
|
| 407 |
+
--model.action-exec-horizon=$ACTION_EXEC_HORIZON \
|
| 408 |
+
--model.observation-horizon=$STATE_HORIZON \
|
| 409 |
+
--model.odim=39 \
|
| 410 |
+
--model.view_feature_dim=2048 \
|
| 411 |
+
--model.use-future-progress-embedding \
|
| 412 |
+
--model.future-progress-dim=$PROGRESS_DIM \
|
| 413 |
+
--model.tune-vlm \
|
| 414 |
+
--model.lang-backbone-lr=1e-5 \
|
| 415 |
+
--model.no-use_film \
|
| 416 |
+
--model.no-combined_temb \
|
| 417 |
+
--model.rtc \
|
| 418 |
+
--model.max-delay=11 \
|
| 419 |
+
--train.max-grad-value=1 \
|
| 420 |
+
--train.loss-group-weight.joint-abs=1.0 \
|
| 421 |
+
--train.loss-group-weight.root-delta-xy=3.0 \
|
| 422 |
+
--train.loss-group-weight.root-rot5-delta=10.0 \
|
| 423 |
+
--train.loss-group-weight.root-height=15.0 \
|
| 424 |
+
--train.loss-group-weight.foot-contact=0.1
|
| 425 |
+
"
|
| 426 |
+
torchrun --nproc_per_node=$NPROC_PER_NODE --master_port=29500 scripts/train.py \
|
| 427 |
+
$args "$@"
|
posttrain-v1-psi0-all-7task-tune-vlm-direct-dense.sh
ADDED
|
@@ -0,0 +1,305 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
pkill -f monitor.py
|
| 2 |
+
|
| 3 |
+
PROJECT_ROOT=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
|
| 4 |
+
cd "$PROJECT_ROOT"
|
| 5 |
+
export PYTHONPATH="$PROJECT_ROOT/src${PYTHONPATH:+:$PYTHONPATH}"
|
| 6 |
+
export CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7
|
| 7 |
+
export PSI_OUTPUT_DIR=${PSI_OUTPUT_DIR:-$PROJECT_ROOT/runs}
|
| 8 |
+
export PSI_LOG_REPORT_TO=wandb
|
| 9 |
+
|
| 10 |
+
source "$PROJECT_ROOT/scripts/activate_psi_env.sh"
|
| 11 |
+
[ -f .env ] && source .env
|
| 12 |
+
|
| 13 |
+
|
| 14 |
+
export OMP_NUM_THREADS=8
|
| 15 |
+
export TF_CPP_MIN_LOG_LEVEL=3
|
| 16 |
+
|
| 17 |
+
# Keep JSON sources readable in this script, but pass compact JSON to tyro.
|
| 18 |
+
compact_json() {
|
| 19 |
+
python -c 'import json, sys; print(json.dumps(json.load(sys.stdin), separators=(",", ":")))'
|
| 20 |
+
}
|
| 21 |
+
|
| 22 |
+
PSI_SEED_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/Psi0/dataset/seed_all
|
| 23 |
+
PSI_SEED_METADATA=metadata/seed_metadata_v004.csv
|
| 24 |
+
|
| 25 |
+
PSI_DANCE_MOTION_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/GR00T-WholeBodyControl/gear_sonic_deploy/reference/converted/psi0_dance/gvhmr
|
| 26 |
+
PSI_DANCE_AUDIO_VIDEO_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/dance_1723_video_psi0/dance_1723_video_10s
|
| 27 |
+
PSI_DANCE_VIDEO_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/dance_1723_video_psi0/dance_1723_video_10s
|
| 28 |
+
PSI_DANCE_VIDEO_EMBED_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/dance_1723_video_psi0/dance_1723_video_10s_embeds
|
| 29 |
+
PSI_VIDEO_DEMO_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/videogen/motion_example_0506_demo
|
| 30 |
+
PSI_QWEN3_TTS_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/qwen3-tts
|
| 31 |
+
PSI_VIDEO_CLIP_DATASET_ROOT=${PSI_VIDEO_CLIP_DATASET_ROOT:-/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/videogen/psi0_video_clip_50f_dense_stride1_direct_full}
|
| 32 |
+
PSI_DANCE_VIDEO_CLIP_ROOT=$PSI_VIDEO_CLIP_DATASET_ROOT/dance_1723_video_10s
|
| 33 |
+
PSI_MOTION_EXAMPLE_VIDEO_CLIP_ROOT=$PSI_VIDEO_CLIP_DATASET_ROOT/motion_example_0506_demo
|
| 34 |
+
PSI_DANCE_VIDEO_CLIP_EMBED_ROOT=${PSI_DANCE_VIDEO_CLIP_EMBED_ROOT:-$PSI_DANCE_VIDEO_CLIP_ROOT/embeds}
|
| 35 |
+
PSI_MOTION_EXAMPLE_VIDEO_CLIP_EMBED_ROOT=${PSI_MOTION_EXAMPLE_VIDEO_CLIP_EMBED_ROOT:-$PSI_MOTION_EXAMPLE_VIDEO_CLIP_ROOT/embeds}
|
| 36 |
+
PSI_MOTION_EXAMPLE_VIDEO_EMBED_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/videogen/motion_example_0506_demo_video_embeds
|
| 37 |
+
|
| 38 |
+
if [ -z "$PSI_AUDIO_SOURCES_JSON" ]; then
|
| 39 |
+
PSI_AUDIO_SOURCES_JSON=$(compact_json <<JSON
|
| 40 |
+
[
|
| 41 |
+
{
|
| 42 |
+
"name": "psi0_dance_audio",
|
| 43 |
+
"root_dir": "$PSI_DANCE_MOTION_ROOT",
|
| 44 |
+
"video_dir": "$PSI_DANCE_AUDIO_VIDEO_ROOT",
|
| 45 |
+
"dataset_name": "seed-audio-g1"
|
| 46 |
+
}
|
| 47 |
+
]
|
| 48 |
+
JSON
|
| 49 |
+
)
|
| 50 |
+
fi
|
| 51 |
+
|
| 52 |
+
if [ -z "$PSI_AUDIO_COMMAND_SOURCES_JSON" ]; then
|
| 53 |
+
PSI_AUDIO_COMMAND_SOURCES_JSON=$(compact_json <<JSON
|
| 54 |
+
[
|
| 55 |
+
{
|
| 56 |
+
"name": "qwen3_tts_seed_audio_command",
|
| 57 |
+
"root_dir": "$PSI_SEED_ROOT",
|
| 58 |
+
"dataset_root": "$PSI_QWEN3_TTS_ROOT",
|
| 59 |
+
"metadata_jsonl": "metadata.jsonl",
|
| 60 |
+
"motion_dir_key": "filename",
|
| 61 |
+
"audio_path_key": "audio_path",
|
| 62 |
+
"instruction_key": "content_short_description",
|
| 63 |
+
"motion_metadata_csv": "$PSI_SEED_METADATA",
|
| 64 |
+
"motion_metadata_key": "filename",
|
| 65 |
+
"motion_metadata_path_key": "move_g1_path",
|
| 66 |
+
"audio_file_extensions": ".wav",
|
| 67 |
+
"audio_task": "command",
|
| 68 |
+
"dataset_name": "seed-audio-command-g1"
|
| 69 |
+
}
|
| 70 |
+
]
|
| 71 |
+
JSON
|
| 72 |
+
)
|
| 73 |
+
fi
|
| 74 |
+
|
| 75 |
+
if [ -z "$PSI_VIDEO_SOURCES_JSON" ]; then
|
| 76 |
+
PSI_VIDEO_SOURCES_JSON=$(compact_json <<JSON
|
| 77 |
+
[
|
| 78 |
+
{
|
| 79 |
+
"name": "psi0_dance_gvhmr",
|
| 80 |
+
"root_dir": "$PSI_DANCE_MOTION_ROOT",
|
| 81 |
+
"video_dir": "$PSI_DANCE_VIDEO_ROOT",
|
| 82 |
+
"video_embed_dir": "$PSI_DANCE_VIDEO_EMBED_ROOT",
|
| 83 |
+
"dataset_name": "seed-video-g1"
|
| 84 |
+
},
|
| 85 |
+
{
|
| 86 |
+
"name": "motion_example_0506_demo",
|
| 87 |
+
"dataset_root": "$PSI_VIDEO_DEMO_ROOT",
|
| 88 |
+
"metadata_jsonl": "metadata.jsonl",
|
| 89 |
+
"motion_dir_key": "converted_path",
|
| 90 |
+
"video_path_key": "video_path",
|
| 91 |
+
"instruction_key": "video_caption",
|
| 92 |
+
"video_embed_dir": "$PSI_MOTION_EXAMPLE_VIDEO_EMBED_ROOT",
|
| 93 |
+
"dataset_name": "motion-example-video-g1"
|
| 94 |
+
}
|
| 95 |
+
]
|
| 96 |
+
JSON
|
| 97 |
+
)
|
| 98 |
+
fi
|
| 99 |
+
|
| 100 |
+
if [ -z "$PSI_VIDEO_CLIP_SOURCES_JSON" ]; then
|
| 101 |
+
PSI_VIDEO_CLIP_SOURCES_JSON=$(compact_json <<JSON
|
| 102 |
+
[
|
| 103 |
+
{
|
| 104 |
+
"name": "dance_1723_video_10s_clip50",
|
| 105 |
+
"dataset_root": "$PSI_DANCE_VIDEO_CLIP_ROOT",
|
| 106 |
+
"metadata_jsonl": "metadata.jsonl",
|
| 107 |
+
"motion_dir_key": "converted_path",
|
| 108 |
+
"video_path_key": "video_path",
|
| 109 |
+
"instruction_key": "video_caption",
|
| 110 |
+
"video_embed_dir": "$PSI_DANCE_VIDEO_CLIP_EMBED_ROOT",
|
| 111 |
+
"dataset_name": "dance-video-clip-g1"
|
| 112 |
+
},
|
| 113 |
+
{
|
| 114 |
+
"name": "motion_example_0506_demo_clip50",
|
| 115 |
+
"dataset_root": "$PSI_MOTION_EXAMPLE_VIDEO_CLIP_ROOT",
|
| 116 |
+
"metadata_jsonl": "metadata.jsonl",
|
| 117 |
+
"motion_dir_key": "converted_path",
|
| 118 |
+
"video_path_key": "video_path",
|
| 119 |
+
"instruction_key": "video_caption",
|
| 120 |
+
"video_embed_dir": "$PSI_MOTION_EXAMPLE_VIDEO_CLIP_EMBED_ROOT",
|
| 121 |
+
"dataset_name": "motion-example-video-clip-g1"
|
| 122 |
+
}
|
| 123 |
+
]
|
| 124 |
+
JSON
|
| 125 |
+
)
|
| 126 |
+
fi
|
| 127 |
+
|
| 128 |
+
# 7-task from-scratch mix, excluding legacy VIDEO_TEXT / VIDEO_ONLY.
|
| 129 |
+
# Smaller data sources get larger weights. These use a tempered inverse-size
|
| 130 |
+
# rule, (47M / source_size)^0.25, so rare modalities are upweighted without
|
| 131 |
+
# completely starving text/video_clip in BatchMixtureSampler.
|
| 132 |
+
# text stride1 frame samples: ~47,000,000
|
| 133 |
+
# music audio dance frame samples: 213,731
|
| 134 |
+
# audio_command wav files: 41,760
|
| 135 |
+
# dense video_clip metadata: 4,597,576
|
| 136 |
+
PSI_TEXT_TASK_WEIGHT=1.0
|
| 137 |
+
PSI_AUDIO_TEXT_TASK_WEIGHT=3.8509
|
| 138 |
+
PSI_AUDIO_ONLY_TASK_WEIGHT=3.8509
|
| 139 |
+
PSI_AUDIO_COMMAND_TEXT_TASK_WEIGHT=5.7921
|
| 140 |
+
PSI_AUDIO_COMMAND_ONLY_TASK_WEIGHT=5.7921
|
| 141 |
+
PSI_VIDEO_TEXT_TASK_WEIGHT=0
|
| 142 |
+
PSI_VIDEO_ONLY_TASK_WEIGHT=0
|
| 143 |
+
PSI_VIDEO_CLIP_TEXT_TASK_WEIGHT=1.7881
|
| 144 |
+
PSI_VIDEO_CLIP_ONLY_TASK_WEIGHT=1.7881
|
| 145 |
+
|
| 146 |
+
PSI_PRETRAIN_BACKBONE_PATH=/cpfs01/VideoGen/public_ckpt/Qwen/Qwen2.5-Omni-3B
|
| 147 |
+
NUM_PAST_FRAMES=0
|
| 148 |
+
STATE_HORIZON=$((NUM_PAST_FRAMES + 1))
|
| 149 |
+
ACTION_CHUNK_SIZE=30
|
| 150 |
+
ACTION_EXEC_HORIZON=20
|
| 151 |
+
RTC_PREFIX_FRAMES=$((ACTION_CHUNK_SIZE - ACTION_EXEC_HORIZON))
|
| 152 |
+
PROGRESS_DIM=5
|
| 153 |
+
VIDEO_SAMPLE_FPS=1.0
|
| 154 |
+
VIDEO_CLIP_SAMPLE_FPS=8.0
|
| 155 |
+
VIDEO_SIZE_PIXELS=50176
|
| 156 |
+
VIDEO_DECODER=ffmpeg
|
| 157 |
+
VIDEO_CACHE_SIZE=64
|
| 158 |
+
VIDEO_CLIP_MOTION_CACHE_SIZE=256
|
| 159 |
+
VIDEO_CLIP_TRUST_METADATA=${VIDEO_CLIP_TRUST_METADATA:-1}
|
| 160 |
+
VIDEO_CLIP_RECORDS_CACHE_DIR=${VIDEO_CLIP_RECORDS_CACHE_DIR:-$PSI_VIDEO_CLIP_DATASET_ROOT/.index_cache}
|
| 161 |
+
if [ "$VIDEO_CLIP_TRUST_METADATA" = "1" ] || [ "$VIDEO_CLIP_TRUST_METADATA" = "true" ] || [ "$VIDEO_CLIP_TRUST_METADATA" = "True" ]; then
|
| 162 |
+
VIDEO_CLIP_TRUST_METADATA_FLAG="--data.video-clip.trust-metadata"
|
| 163 |
+
else
|
| 164 |
+
VIDEO_CLIP_TRUST_METADATA_FLAG="--data.video-clip.no-trust-metadata"
|
| 165 |
+
fi
|
| 166 |
+
DATALOADER_NUM_WORKERS=4
|
| 167 |
+
VAL_DATALOADER_NUM_WORKERS=2
|
| 168 |
+
|
| 169 |
+
NPROC_PER_NODE=$(echo $CUDA_VISIBLE_DEVICES | tr ',' '\n' | wc -l)
|
| 170 |
+
|
| 171 |
+
echo "Training with $NPROC_PER_NODE GPU(s): $CUDA_VISIBLE_DEVICES"
|
| 172 |
+
echo "Text root: $PSI_SEED_ROOT"
|
| 173 |
+
echo "Dance motion root: $PSI_DANCE_MOTION_ROOT"
|
| 174 |
+
echo "Dance audio/video root: $PSI_DANCE_AUDIO_VIDEO_ROOT"
|
| 175 |
+
echo "Dance video root: $PSI_DANCE_VIDEO_ROOT"
|
| 176 |
+
echo "Dance video embed root: $PSI_DANCE_VIDEO_EMBED_ROOT"
|
| 177 |
+
echo "Qwen3-TTS acdudio command root: $PSI_QWEN3_TTS_ROOT"
|
| 178 |
+
echo "Audio sources: $PSI_AUDIO_SOURCES_JSON"
|
| 179 |
+
echo "Audio command sources: $PSI_AUDIO_COMMAND_SOURCES_JSON"
|
| 180 |
+
echo "Video sources: $PSI_VIDEO_SOURCES_JSON"
|
| 181 |
+
echo "Video clip sources: $PSI_VIDEO_CLIP_SOURCES_JSON"
|
| 182 |
+
echo "Task weights: text=$PSI_TEXT_TASK_WEIGHT audio_text=$PSI_AUDIO_TEXT_TASK_WEIGHT audio_only=$PSI_AUDIO_ONLY_TASK_WEIGHT audio_command_text=$PSI_AUDIO_COMMAND_TEXT_TASK_WEIGHT audio_command_only=$PSI_AUDIO_COMMAND_ONLY_TASK_WEIGHT video_text=$PSI_VIDEO_TEXT_TASK_WEIGHT video_only=$PSI_VIDEO_ONLY_TASK_WEIGHT video_clip_text=$PSI_VIDEO_CLIP_TEXT_TASK_WEIGHT video_clip_only=$PSI_VIDEO_CLIP_ONLY_TASK_WEIGHT"
|
| 183 |
+
# echo "Resume run dir: $PSI_PRETRAIN_RUN_DIR"
|
| 184 |
+
echo "TextOp 39D action/state enabled"
|
| 185 |
+
echo "Future progress embedding: enabled, dim $PROGRESS_DIM"
|
| 186 |
+
echo "Backbone path: $PSI_PRETRAIN_BACKBONE_PATH"
|
| 187 |
+
echo "Output dir: $PSI_OUTPUT_DIR"
|
| 188 |
+
echo "Video clip trust metadata: $VIDEO_CLIP_TRUST_METADATA"
|
| 189 |
+
echo "Video clip records cache dir: $VIDEO_CLIP_RECORDS_CACHE_DIR"
|
| 190 |
+
|
| 191 |
+
args="posttrain_v1_seed_text_audio_video_mix_psi0_config \
|
| 192 |
+
--seed=292285 \
|
| 193 |
+
--exp=posttrain \
|
| 194 |
+
--timestamp=$(date +"%y%m%d%H%M") \
|
| 195 |
+
--train.name=posttrain_v1_trimodal \
|
| 196 |
+
--train.data_parallel=ddp \
|
| 197 |
+
--train.mixed_precision=bf16 \
|
| 198 |
+
--train.train_batch_size=32 \
|
| 199 |
+
--train.val_batch_size=64 \
|
| 200 |
+
--train.num-workers=$DATALOADER_NUM_WORKERS \
|
| 201 |
+
--train.val-num-workers=$VAL_DATALOADER_NUM_WORKERS \
|
| 202 |
+
--train.resume_from_checkpoint=None \
|
| 203 |
+
--train.max_checkpoints_to_keep=1 \
|
| 204 |
+
--train.gradient_accumulation_steps=1 \
|
| 205 |
+
--train.learning_rate=1e-4 \
|
| 206 |
+
--train.max_training_steps=500000 \
|
| 207 |
+
--train.warmup_ratio=None \
|
| 208 |
+
--train.warmup_steps=1000 \
|
| 209 |
+
--train.checkpointing_steps=1000 \
|
| 210 |
+
--train.validation_steps=1000 \
|
| 211 |
+
--train.val_num_batches=20 \
|
| 212 |
+
--train.lr_scheduler_type=cosine_with_min_lr \
|
| 213 |
+
--train.lr_scheduler_kwargs.betas 0.9 0.999 \
|
| 214 |
+
--train.lr_scheduler_kwargs.weight_decay=0 \
|
| 215 |
+
--train.lr_scheduler_kwargs.eps=1e-8 \
|
| 216 |
+
--log.report_to=$PSI_LOG_REPORT_TO \
|
| 217 |
+
--train.output_dir=$PSI_OUTPUT_DIR \
|
| 218 |
+
--data.text.root-dir=$PSI_SEED_ROOT \
|
| 219 |
+
--data.text.metadata-csv=$PSI_SEED_METADATA \
|
| 220 |
+
--data.text-ratio=$PSI_TEXT_TASK_WEIGHT \
|
| 221 |
+
--data.audio.sources-json=$PSI_AUDIO_SOURCES_JSON \
|
| 222 |
+
--data.audio-ratio=$PSI_AUDIO_TEXT_TASK_WEIGHT \
|
| 223 |
+
--data.audio-only-ratio=$PSI_AUDIO_ONLY_TASK_WEIGHT \
|
| 224 |
+
--data.audio.sample-stride=1 \
|
| 225 |
+
--data.audio.motion-fps=50 \
|
| 226 |
+
--data.audio.audio-sampling-rate=16000 \
|
| 227 |
+
--data.audio.audio-window-seconds=1.0 \
|
| 228 |
+
--data.audio.audio-left-context-seconds=0.0 \
|
| 229 |
+
--data.audio.full-audio \
|
| 230 |
+
--data.audio-command.sources-json=$PSI_AUDIO_COMMAND_SOURCES_JSON \
|
| 231 |
+
--data.audio-command-ratio=$PSI_AUDIO_COMMAND_TEXT_TASK_WEIGHT \
|
| 232 |
+
--data.audio-command-only-ratio=$PSI_AUDIO_COMMAND_ONLY_TASK_WEIGHT \
|
| 233 |
+
--data.audio-command.sample-stride=1 \
|
| 234 |
+
--data.audio-command.motion-fps=50 \
|
| 235 |
+
--data.audio-command.audio-sampling-rate=16000 \
|
| 236 |
+
--data.audio-command.audio-window-seconds=1.0 \
|
| 237 |
+
--data.audio-command.audio-left-context-seconds=0.0 \
|
| 238 |
+
--data.audio-command.full-audio \
|
| 239 |
+
--data.video-ratio=$PSI_VIDEO_TEXT_TASK_WEIGHT \
|
| 240 |
+
--data.video-only-ratio=$PSI_VIDEO_ONLY_TASK_WEIGHT \
|
| 241 |
+
--data.video.sample-stride=1 \
|
| 242 |
+
--data.video.motion-fps=50 \
|
| 243 |
+
--data.video.video-sample-fps=$VIDEO_SAMPLE_FPS \
|
| 244 |
+
--data.video.video-size-pixels=$VIDEO_SIZE_PIXELS \
|
| 245 |
+
--data.video.video-decoder=$VIDEO_DECODER \
|
| 246 |
+
--data.video.video-cache-size=$VIDEO_CACHE_SIZE \
|
| 247 |
+
--data.video.sources-json=$PSI_VIDEO_SOURCES_JSON \
|
| 248 |
+
--data.video-clip-text-ratio=$PSI_VIDEO_CLIP_TEXT_TASK_WEIGHT \
|
| 249 |
+
--data.video-clip-only-ratio=$PSI_VIDEO_CLIP_ONLY_TASK_WEIGHT \
|
| 250 |
+
--data.video-clip.sources-json=$PSI_VIDEO_CLIP_SOURCES_JSON \
|
| 251 |
+
--data.video-clip.sample-stride=1 \
|
| 252 |
+
--data.video-clip.motion-fps=50 \
|
| 253 |
+
--data.video-clip.video-sample-fps=$VIDEO_CLIP_SAMPLE_FPS \
|
| 254 |
+
--data.video-clip.video-size-pixels=$VIDEO_SIZE_PIXELS \
|
| 255 |
+
--data.video-clip.video-decoder=$VIDEO_DECODER \
|
| 256 |
+
--data.video-clip.video-cache-size=$VIDEO_CACHE_SIZE \
|
| 257 |
+
--data.video-clip.motion-cache-size=$VIDEO_CLIP_MOTION_CACHE_SIZE \
|
| 258 |
+
$VIDEO_CLIP_TRUST_METADATA_FLAG \
|
| 259 |
+
--data.video-clip.records-cache-dir=$VIDEO_CLIP_RECORDS_CACHE_DIR \
|
| 260 |
+
--data.video-clip.action-exec-horizon=$ACTION_EXEC_HORIZON \
|
| 261 |
+
--data.sampler=batch_mixture \
|
| 262 |
+
--data.val-task=text,audio,audio_only,audio_command,audio_command_only,video_clip,video_clip_only \
|
| 263 |
+
--data.transform.repack.num-past-frames=$NUM_PAST_FRAMES \
|
| 264 |
+
--data.transform.repack.textop-actions \
|
| 265 |
+
--data.transform.repack.action-chunk-size=$ACTION_CHUNK_SIZE \
|
| 266 |
+
--data.transform.repack.rtc-prefix-frames=$RTC_PREFIX_FRAMES \
|
| 267 |
+
--data.transform.repack.pad-action-dim=39 \
|
| 268 |
+
--data.transform.repack.pad-state-dim=39 \
|
| 269 |
+
--data.transform.field.stat-path=assets/stats/seed_all_g1_motion39_textop_stats.json \
|
| 270 |
+
--data.transform.field.normalize-state \
|
| 271 |
+
--data.transform.field.pad-action-dim=39 \
|
| 272 |
+
--data.transform.field.pad-state-dim=39 \
|
| 273 |
+
--data.transform.model.resize.size 240 320 \
|
| 274 |
+
--data.transform.model.center_crop.size 240 320 \
|
| 275 |
+
--data.transform.model.no-img-aug \
|
| 276 |
+
--data.transform.model.text-only \
|
| 277 |
+
--data.transform.model.audio-only \
|
| 278 |
+
--data.transform.model.video-only \
|
| 279 |
+
--data.transform.model.video-size-pixels=$VIDEO_SIZE_PIXELS \
|
| 280 |
+
--model.model_name_or_path=$PSI_PRETRAIN_BACKBONE_PATH \
|
| 281 |
+
--model.noise-scheduler=flow \
|
| 282 |
+
--model.n_conditions=0 \
|
| 283 |
+
--model.action-chunk-size=$ACTION_CHUNK_SIZE \
|
| 284 |
+
--model.action-dim=39 \
|
| 285 |
+
--model.action-exec-horizon=$ACTION_EXEC_HORIZON \
|
| 286 |
+
--model.observation-horizon=$STATE_HORIZON \
|
| 287 |
+
--model.odim=39 \
|
| 288 |
+
--model.view_feature_dim=2048 \
|
| 289 |
+
--model.use-future-progress-embedding \
|
| 290 |
+
--model.future-progress-dim=$PROGRESS_DIM \
|
| 291 |
+
--model.tune-vlm \
|
| 292 |
+
--model.lang-backbone-lr=1e-5 \
|
| 293 |
+
--model.no-use_film \
|
| 294 |
+
--model.no-combined_temb \
|
| 295 |
+
--model.rtc \
|
| 296 |
+
--model.max-delay=11 \
|
| 297 |
+
--train.max-grad-value=1 \
|
| 298 |
+
--train.loss-group-weight.joint-abs=1.0 \
|
| 299 |
+
--train.loss-group-weight.root-delta-xy=3.0 \
|
| 300 |
+
--train.loss-group-weight.root-rot5-delta=10.0 \
|
| 301 |
+
--train.loss-group-weight.root-height=15.0 \
|
| 302 |
+
--train.loss-group-weight.foot-contact=0.1
|
| 303 |
+
"
|
| 304 |
+
torchrun --nproc_per_node=$NPROC_PER_NODE --master_port=29500 scripts/train.py \
|
| 305 |
+
$args "$@"
|
posttrain-v1-psi0-all-tune-vlm-direct-dense.sh
ADDED
|
@@ -0,0 +1,297 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
pkill -f monitor.py
|
| 2 |
+
|
| 3 |
+
PROJECT_ROOT=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
|
| 4 |
+
cd "$PROJECT_ROOT"
|
| 5 |
+
export PYTHONPATH="$PROJECT_ROOT/src${PYTHONPATH:+:$PYTHONPATH}"
|
| 6 |
+
export CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7
|
| 7 |
+
export PSI_OUTPUT_DIR=${PSI_OUTPUT_DIR:-$PROJECT_ROOT/runs}
|
| 8 |
+
export PSI_LOG_REPORT_TO=wandb
|
| 9 |
+
|
| 10 |
+
source "$PROJECT_ROOT/scripts/activate_psi_env.sh"
|
| 11 |
+
[ -f .env ] && source .env
|
| 12 |
+
|
| 13 |
+
|
| 14 |
+
export OMP_NUM_THREADS=8
|
| 15 |
+
export TF_CPP_MIN_LOG_LEVEL=3
|
| 16 |
+
|
| 17 |
+
# Keep JSON sources readable in this script, but pass compact JSON to tyro.
|
| 18 |
+
compact_json() {
|
| 19 |
+
python -c 'import json, sys; print(json.dumps(json.load(sys.stdin), separators=(",", ":")))'
|
| 20 |
+
}
|
| 21 |
+
|
| 22 |
+
PSI_SEED_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/Psi0/dataset/seed_all
|
| 23 |
+
PSI_SEED_METADATA=metadata/seed_metadata_v004.csv
|
| 24 |
+
|
| 25 |
+
PSI_DANCE_MOTION_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/GR00T-WholeBodyControl/gear_sonic_deploy/reference/converted/psi0_dance/gvhmr
|
| 26 |
+
PSI_DANCE_AUDIO_VIDEO_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/dance_1723_video_psi0/dance_1723_video_10s
|
| 27 |
+
PSI_DANCE_VIDEO_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/dance_1723_video_psi0/dance_1723_video_10s
|
| 28 |
+
PSI_DANCE_VIDEO_EMBED_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/dance_1723_video_psi0/dance_1723_video_10s_embeds
|
| 29 |
+
PSI_VIDEO_DEMO_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/videogen/motion_example_0506_demo
|
| 30 |
+
PSI_QWEN3_TTS_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/qwen3-tts
|
| 31 |
+
PSI_VIDEO_CLIP_DATASET_ROOT=${PSI_VIDEO_CLIP_DATASET_ROOT:-/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/videogen/psi0_video_clip_50f_dense_stride1_direct_full}
|
| 32 |
+
PSI_DANCE_VIDEO_CLIP_ROOT=$PSI_VIDEO_CLIP_DATASET_ROOT/dance_1723_video_10s
|
| 33 |
+
PSI_MOTION_EXAMPLE_VIDEO_CLIP_ROOT=$PSI_VIDEO_CLIP_DATASET_ROOT/motion_example_0506_demo
|
| 34 |
+
PSI_DANCE_VIDEO_CLIP_EMBED_ROOT=${PSI_DANCE_VIDEO_CLIP_EMBED_ROOT:-$PSI_DANCE_VIDEO_CLIP_ROOT/embeds}
|
| 35 |
+
PSI_MOTION_EXAMPLE_VIDEO_CLIP_EMBED_ROOT=${PSI_MOTION_EXAMPLE_VIDEO_CLIP_EMBED_ROOT:-$PSI_MOTION_EXAMPLE_VIDEO_CLIP_ROOT/embeds}
|
| 36 |
+
PSI_MOTION_EXAMPLE_VIDEO_EMBED_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/videogen/motion_example_0506_demo_video_embeds
|
| 37 |
+
|
| 38 |
+
if [ -z "$PSI_AUDIO_SOURCES_JSON" ]; then
|
| 39 |
+
PSI_AUDIO_SOURCES_JSON=$(compact_json <<JSON
|
| 40 |
+
[
|
| 41 |
+
{
|
| 42 |
+
"name": "psi0_dance_audio",
|
| 43 |
+
"root_dir": "$PSI_DANCE_MOTION_ROOT",
|
| 44 |
+
"video_dir": "$PSI_DANCE_AUDIO_VIDEO_ROOT",
|
| 45 |
+
"dataset_name": "seed-audio-g1"
|
| 46 |
+
}
|
| 47 |
+
]
|
| 48 |
+
JSON
|
| 49 |
+
)
|
| 50 |
+
fi
|
| 51 |
+
|
| 52 |
+
if [ -z "$PSI_AUDIO_COMMAND_SOURCES_JSON" ]; then
|
| 53 |
+
PSI_AUDIO_COMMAND_SOURCES_JSON=$(compact_json <<JSON
|
| 54 |
+
[
|
| 55 |
+
{
|
| 56 |
+
"name": "qwen3_tts_seed_audio_command",
|
| 57 |
+
"root_dir": "$PSI_SEED_ROOT",
|
| 58 |
+
"dataset_root": "$PSI_QWEN3_TTS_ROOT",
|
| 59 |
+
"metadata_jsonl": "metadata.jsonl",
|
| 60 |
+
"motion_dir_key": "filename",
|
| 61 |
+
"audio_path_key": "audio_path",
|
| 62 |
+
"instruction_key": "content_short_description",
|
| 63 |
+
"motion_metadata_csv": "$PSI_SEED_METADATA",
|
| 64 |
+
"motion_metadata_key": "filename",
|
| 65 |
+
"motion_metadata_path_key": "move_g1_path",
|
| 66 |
+
"audio_file_extensions": ".wav",
|
| 67 |
+
"audio_task": "command",
|
| 68 |
+
"dataset_name": "seed-audio-command-g1"
|
| 69 |
+
}
|
| 70 |
+
]
|
| 71 |
+
JSON
|
| 72 |
+
)
|
| 73 |
+
fi
|
| 74 |
+
|
| 75 |
+
if [ -z "$PSI_VIDEO_SOURCES_JSON" ]; then
|
| 76 |
+
PSI_VIDEO_SOURCES_JSON=$(compact_json <<JSON
|
| 77 |
+
[
|
| 78 |
+
{
|
| 79 |
+
"name": "psi0_dance_gvhmr",
|
| 80 |
+
"root_dir": "$PSI_DANCE_MOTION_ROOT",
|
| 81 |
+
"video_dir": "$PSI_DANCE_VIDEO_ROOT",
|
| 82 |
+
"video_embed_dir": "$PSI_DANCE_VIDEO_EMBED_ROOT",
|
| 83 |
+
"dataset_name": "seed-video-g1"
|
| 84 |
+
},
|
| 85 |
+
{
|
| 86 |
+
"name": "motion_example_0506_demo",
|
| 87 |
+
"dataset_root": "$PSI_VIDEO_DEMO_ROOT",
|
| 88 |
+
"metadata_jsonl": "metadata.jsonl",
|
| 89 |
+
"motion_dir_key": "converted_path",
|
| 90 |
+
"video_path_key": "video_path",
|
| 91 |
+
"instruction_key": "video_caption",
|
| 92 |
+
"video_embed_dir": "$PSI_MOTION_EXAMPLE_VIDEO_EMBED_ROOT",
|
| 93 |
+
"dataset_name": "motion-example-video-g1"
|
| 94 |
+
}
|
| 95 |
+
]
|
| 96 |
+
JSON
|
| 97 |
+
)
|
| 98 |
+
fi
|
| 99 |
+
|
| 100 |
+
if [ -z "$PSI_VIDEO_CLIP_SOURCES_JSON" ]; then
|
| 101 |
+
PSI_VIDEO_CLIP_SOURCES_JSON=$(compact_json <<JSON
|
| 102 |
+
[
|
| 103 |
+
{
|
| 104 |
+
"name": "dance_1723_video_10s_clip50",
|
| 105 |
+
"dataset_root": "$PSI_DANCE_VIDEO_CLIP_ROOT",
|
| 106 |
+
"metadata_jsonl": "metadata.jsonl",
|
| 107 |
+
"motion_dir_key": "converted_path",
|
| 108 |
+
"video_path_key": "video_path",
|
| 109 |
+
"instruction_key": "video_caption",
|
| 110 |
+
"video_embed_dir": "$PSI_DANCE_VIDEO_CLIP_EMBED_ROOT",
|
| 111 |
+
"dataset_name": "dance-video-clip-g1"
|
| 112 |
+
},
|
| 113 |
+
{
|
| 114 |
+
"name": "motion_example_0506_demo_clip50",
|
| 115 |
+
"dataset_root": "$PSI_MOTION_EXAMPLE_VIDEO_CLIP_ROOT",
|
| 116 |
+
"metadata_jsonl": "metadata.jsonl",
|
| 117 |
+
"motion_dir_key": "converted_path",
|
| 118 |
+
"video_path_key": "video_path",
|
| 119 |
+
"instruction_key": "video_caption",
|
| 120 |
+
"video_embed_dir": "$PSI_MOTION_EXAMPLE_VIDEO_CLIP_EMBED_ROOT",
|
| 121 |
+
"dataset_name": "motion-example-video-clip-g1"
|
| 122 |
+
}
|
| 123 |
+
]
|
| 124 |
+
JSON
|
| 125 |
+
)
|
| 126 |
+
fi
|
| 127 |
+
|
| 128 |
+
PSI_TEXT_TASK_WEIGHT=0
|
| 129 |
+
PSI_AUDIO_TEXT_TASK_WEIGHT=0
|
| 130 |
+
PSI_AUDIO_ONLY_TASK_WEIGHT=0
|
| 131 |
+
PSI_AUDIO_COMMAND_TEXT_TASK_WEIGHT=0
|
| 132 |
+
PSI_AUDIO_COMMAND_ONLY_TASK_WEIGHT=0
|
| 133 |
+
PSI_VIDEO_TEXT_TASK_WEIGHT=0
|
| 134 |
+
PSI_VIDEO_ONLY_TASK_WEIGHT=0
|
| 135 |
+
PSI_VIDEO_CLIP_TEXT_TASK_WEIGHT=0.5
|
| 136 |
+
PSI_VIDEO_CLIP_ONLY_TASK_WEIGHT=0.5
|
| 137 |
+
|
| 138 |
+
PSI_PRETRAIN_BACKBONE_PATH=/cpfs01/VideoGen/public_ckpt/Qwen/Qwen2.5-Omni-3B
|
| 139 |
+
NUM_PAST_FRAMES=0
|
| 140 |
+
STATE_HORIZON=$((NUM_PAST_FRAMES + 1))
|
| 141 |
+
ACTION_CHUNK_SIZE=30
|
| 142 |
+
ACTION_EXEC_HORIZON=20
|
| 143 |
+
RTC_PREFIX_FRAMES=$((ACTION_CHUNK_SIZE - ACTION_EXEC_HORIZON))
|
| 144 |
+
PROGRESS_DIM=5
|
| 145 |
+
VIDEO_SAMPLE_FPS=1.0
|
| 146 |
+
VIDEO_CLIP_SAMPLE_FPS=8.0
|
| 147 |
+
VIDEO_SIZE_PIXELS=50176
|
| 148 |
+
VIDEO_DECODER=ffmpeg
|
| 149 |
+
VIDEO_CACHE_SIZE=64
|
| 150 |
+
VIDEO_CLIP_MOTION_CACHE_SIZE=256
|
| 151 |
+
VIDEO_CLIP_TRUST_METADATA=${VIDEO_CLIP_TRUST_METADATA:-1}
|
| 152 |
+
VIDEO_CLIP_RECORDS_CACHE_DIR=${VIDEO_CLIP_RECORDS_CACHE_DIR:-$PSI_VIDEO_CLIP_DATASET_ROOT/.index_cache}
|
| 153 |
+
if [ "$VIDEO_CLIP_TRUST_METADATA" = "1" ] || [ "$VIDEO_CLIP_TRUST_METADATA" = "true" ] || [ "$VIDEO_CLIP_TRUST_METADATA" = "True" ]; then
|
| 154 |
+
VIDEO_CLIP_TRUST_METADATA_FLAG="--data.video-clip.trust-metadata"
|
| 155 |
+
else
|
| 156 |
+
VIDEO_CLIP_TRUST_METADATA_FLAG="--data.video-clip.no-trust-metadata"
|
| 157 |
+
fi
|
| 158 |
+
DATALOADER_NUM_WORKERS=4
|
| 159 |
+
VAL_DATALOADER_NUM_WORKERS=2
|
| 160 |
+
|
| 161 |
+
NPROC_PER_NODE=$(echo $CUDA_VISIBLE_DEVICES | tr ',' '\n' | wc -l)
|
| 162 |
+
|
| 163 |
+
echo "Training with $NPROC_PER_NODE GPU(s): $CUDA_VISIBLE_DEVICES"
|
| 164 |
+
echo "Text root: $PSI_SEED_ROOT"
|
| 165 |
+
echo "Dance motion root: $PSI_DANCE_MOTION_ROOT"
|
| 166 |
+
echo "Dance audio/video root: $PSI_DANCE_AUDIO_VIDEO_ROOT"
|
| 167 |
+
echo "Dance video root: $PSI_DANCE_VIDEO_ROOT"
|
| 168 |
+
echo "Dance video embed root: $PSI_DANCE_VIDEO_EMBED_ROOT"
|
| 169 |
+
echo "Qwen3-TTS acdudio command root: $PSI_QWEN3_TTS_ROOT"
|
| 170 |
+
echo "Audio sources: $PSI_AUDIO_SOURCES_JSON"
|
| 171 |
+
echo "Audio command sources: $PSI_AUDIO_COMMAND_SOURCES_JSON"
|
| 172 |
+
echo "Video sources: $PSI_VIDEO_SOURCES_JSON"
|
| 173 |
+
echo "Video clip sources: $PSI_VIDEO_CLIP_SOURCES_JSON"
|
| 174 |
+
echo "Task weights: text=$PSI_TEXT_TASK_WEIGHT audio_text=$PSI_AUDIO_TEXT_TASK_WEIGHT audio_only=$PSI_AUDIO_ONLY_TASK_WEIGHT audio_command_text=$PSI_AUDIO_COMMAND_TEXT_TASK_WEIGHT audio_command_only=$PSI_AUDIO_COMMAND_ONLY_TASK_WEIGHT video_text=$PSI_VIDEO_TEXT_TASK_WEIGHT video_only=$PSI_VIDEO_ONLY_TASK_WEIGHT video_clip_text=$PSI_VIDEO_CLIP_TEXT_TASK_WEIGHT video_clip_only=$PSI_VIDEO_CLIP_ONLY_TASK_WEIGHT"
|
| 175 |
+
# echo "Resume run dir: $PSI_PRETRAIN_RUN_DIR"
|
| 176 |
+
echo "TextOp 39D action/state enabled"
|
| 177 |
+
echo "Future progress embedding: enabled, dim $PROGRESS_DIM"
|
| 178 |
+
echo "Backbone path: $PSI_PRETRAIN_BACKBONE_PATH"
|
| 179 |
+
echo "Output dir: $PSI_OUTPUT_DIR"
|
| 180 |
+
echo "Video clip trust metadata: $VIDEO_CLIP_TRUST_METADATA"
|
| 181 |
+
echo "Video clip records cache dir: $VIDEO_CLIP_RECORDS_CACHE_DIR"
|
| 182 |
+
|
| 183 |
+
args="posttrain_v1_seed_text_audio_video_mix_psi0_config \
|
| 184 |
+
--seed=292285 \
|
| 185 |
+
--exp=posttrain \
|
| 186 |
+
--timestamp=$(date +"%y%m%d%H%M") \
|
| 187 |
+
--train.name=posttrain_v1_trimodal \
|
| 188 |
+
--train.data_parallel=ddp \
|
| 189 |
+
--train.mixed_precision=bf16 \
|
| 190 |
+
--train.train_batch_size=32 \
|
| 191 |
+
--train.val_batch_size=64 \
|
| 192 |
+
--train.num-workers=$DATALOADER_NUM_WORKERS \
|
| 193 |
+
--train.val-num-workers=$VAL_DATALOADER_NUM_WORKERS \
|
| 194 |
+
--train.resume_from_checkpoint=None \
|
| 195 |
+
--train.max_checkpoints_to_keep=1 \
|
| 196 |
+
--train.gradient_accumulation_steps=1 \
|
| 197 |
+
--train.learning_rate=1e-4 \
|
| 198 |
+
--train.max_training_steps=500000 \
|
| 199 |
+
--train.warmup_ratio=None \
|
| 200 |
+
--train.warmup_steps=1000 \
|
| 201 |
+
--train.checkpointing_steps=1000 \
|
| 202 |
+
--train.validation_steps=1000 \
|
| 203 |
+
--train.val_num_batches=20 \
|
| 204 |
+
--train.lr_scheduler_type=cosine_with_min_lr \
|
| 205 |
+
--train.lr_scheduler_kwargs.betas 0.9 0.999 \
|
| 206 |
+
--train.lr_scheduler_kwargs.weight_decay=0 \
|
| 207 |
+
--train.lr_scheduler_kwargs.eps=1e-8 \
|
| 208 |
+
--log.report_to=$PSI_LOG_REPORT_TO \
|
| 209 |
+
--train.output_dir=$PSI_OUTPUT_DIR \
|
| 210 |
+
--data.text.root-dir=$PSI_SEED_ROOT \
|
| 211 |
+
--data.text.metadata-csv=$PSI_SEED_METADATA \
|
| 212 |
+
--data.text-ratio=$PSI_TEXT_TASK_WEIGHT \
|
| 213 |
+
--data.audio.sources-json=$PSI_AUDIO_SOURCES_JSON \
|
| 214 |
+
--data.audio-ratio=$PSI_AUDIO_TEXT_TASK_WEIGHT \
|
| 215 |
+
--data.audio-only-ratio=$PSI_AUDIO_ONLY_TASK_WEIGHT \
|
| 216 |
+
--data.audio.sample-stride=1 \
|
| 217 |
+
--data.audio.motion-fps=50 \
|
| 218 |
+
--data.audio.audio-sampling-rate=16000 \
|
| 219 |
+
--data.audio.audio-window-seconds=1.0 \
|
| 220 |
+
--data.audio.audio-left-context-seconds=0.0 \
|
| 221 |
+
--data.audio.full-audio \
|
| 222 |
+
--data.audio-command.sources-json=$PSI_AUDIO_COMMAND_SOURCES_JSON \
|
| 223 |
+
--data.audio-command-ratio=$PSI_AUDIO_COMMAND_TEXT_TASK_WEIGHT \
|
| 224 |
+
--data.audio-command-only-ratio=$PSI_AUDIO_COMMAND_ONLY_TASK_WEIGHT \
|
| 225 |
+
--data.audio-command.sample-stride=1 \
|
| 226 |
+
--data.audio-command.motion-fps=50 \
|
| 227 |
+
--data.audio-command.audio-sampling-rate=16000 \
|
| 228 |
+
--data.audio-command.audio-window-seconds=1.0 \
|
| 229 |
+
--data.audio-command.audio-left-context-seconds=0.0 \
|
| 230 |
+
--data.audio-command.full-audio \
|
| 231 |
+
--data.video-ratio=$PSI_VIDEO_TEXT_TASK_WEIGHT \
|
| 232 |
+
--data.video-only-ratio=$PSI_VIDEO_ONLY_TASK_WEIGHT \
|
| 233 |
+
--data.video.sample-stride=1 \
|
| 234 |
+
--data.video.motion-fps=50 \
|
| 235 |
+
--data.video.video-sample-fps=$VIDEO_SAMPLE_FPS \
|
| 236 |
+
--data.video.video-size-pixels=$VIDEO_SIZE_PIXELS \
|
| 237 |
+
--data.video.video-decoder=$VIDEO_DECODER \
|
| 238 |
+
--data.video.video-cache-size=$VIDEO_CACHE_SIZE \
|
| 239 |
+
--data.video.sources-json=$PSI_VIDEO_SOURCES_JSON \
|
| 240 |
+
--data.video-clip-text-ratio=$PSI_VIDEO_CLIP_TEXT_TASK_WEIGHT \
|
| 241 |
+
--data.video-clip-only-ratio=$PSI_VIDEO_CLIP_ONLY_TASK_WEIGHT \
|
| 242 |
+
--data.video-clip.sources-json=$PSI_VIDEO_CLIP_SOURCES_JSON \
|
| 243 |
+
--data.video-clip.sample-stride=1 \
|
| 244 |
+
--data.video-clip.motion-fps=50 \
|
| 245 |
+
--data.video-clip.video-sample-fps=$VIDEO_CLIP_SAMPLE_FPS \
|
| 246 |
+
--data.video-clip.video-size-pixels=$VIDEO_SIZE_PIXELS \
|
| 247 |
+
--data.video-clip.video-decoder=$VIDEO_DECODER \
|
| 248 |
+
--data.video-clip.video-cache-size=$VIDEO_CACHE_SIZE \
|
| 249 |
+
--data.video-clip.motion-cache-size=$VIDEO_CLIP_MOTION_CACHE_SIZE \
|
| 250 |
+
$VIDEO_CLIP_TRUST_METADATA_FLAG \
|
| 251 |
+
--data.video-clip.records-cache-dir=$VIDEO_CLIP_RECORDS_CACHE_DIR \
|
| 252 |
+
--data.video-clip.action-exec-horizon=$ACTION_EXEC_HORIZON \
|
| 253 |
+
--data.sampler=batch_mixture \
|
| 254 |
+
--data.val-task=video_clip,video_clip_only \
|
| 255 |
+
--data.transform.repack.num-past-frames=$NUM_PAST_FRAMES \
|
| 256 |
+
--data.transform.repack.textop-actions \
|
| 257 |
+
--data.transform.repack.action-chunk-size=$ACTION_CHUNK_SIZE \
|
| 258 |
+
--data.transform.repack.rtc-prefix-frames=$RTC_PREFIX_FRAMES \
|
| 259 |
+
--data.transform.repack.pad-action-dim=39 \
|
| 260 |
+
--data.transform.repack.pad-state-dim=39 \
|
| 261 |
+
--data.transform.field.stat-path=assets/stats/seed_all_g1_motion39_textop_stats.json \
|
| 262 |
+
--data.transform.field.normalize-state \
|
| 263 |
+
--data.transform.field.pad-action-dim=39 \
|
| 264 |
+
--data.transform.field.pad-state-dim=39 \
|
| 265 |
+
--data.transform.model.resize.size 240 320 \
|
| 266 |
+
--data.transform.model.center_crop.size 240 320 \
|
| 267 |
+
--data.transform.model.no-img-aug \
|
| 268 |
+
--data.transform.model.text-only \
|
| 269 |
+
--data.transform.model.audio-only \
|
| 270 |
+
--data.transform.model.video-only \
|
| 271 |
+
--data.transform.model.video-size-pixels=$VIDEO_SIZE_PIXELS \
|
| 272 |
+
--model.model_name_or_path=$PSI_PRETRAIN_BACKBONE_PATH \
|
| 273 |
+
--model.noise-scheduler=flow \
|
| 274 |
+
--model.n_conditions=0 \
|
| 275 |
+
--model.action-chunk-size=$ACTION_CHUNK_SIZE \
|
| 276 |
+
--model.action-dim=39 \
|
| 277 |
+
--model.action-exec-horizon=$ACTION_EXEC_HORIZON \
|
| 278 |
+
--model.observation-horizon=$STATE_HORIZON \
|
| 279 |
+
--model.odim=39 \
|
| 280 |
+
--model.view_feature_dim=2048 \
|
| 281 |
+
--model.use-future-progress-embedding \
|
| 282 |
+
--model.future-progress-dim=$PROGRESS_DIM \
|
| 283 |
+
--model.tune-vlm \
|
| 284 |
+
--model.lang-backbone-lr=1e-5 \
|
| 285 |
+
--model.no-use_film \
|
| 286 |
+
--model.no-combined_temb \
|
| 287 |
+
--model.rtc \
|
| 288 |
+
--model.max-delay=11 \
|
| 289 |
+
--train.max-grad-value=1 \
|
| 290 |
+
--train.loss-group-weight.joint-abs=1.0 \
|
| 291 |
+
--train.loss-group-weight.root-delta-xy=3.0 \
|
| 292 |
+
--train.loss-group-weight.root-rot5-delta=10.0 \
|
| 293 |
+
--train.loss-group-weight.root-height=15.0 \
|
| 294 |
+
--train.loss-group-weight.foot-contact=0.1
|
| 295 |
+
"
|
| 296 |
+
torchrun --nproc_per_node=$NPROC_PER_NODE --master_port=29500 scripts/train.py \
|
| 297 |
+
$args "$@"
|
posttrain-v1-psi0-cache-complete-smoke.sh
ADDED
|
@@ -0,0 +1,447 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#!/usr/bin/env bash
|
| 2 |
+
set -eo pipefail
|
| 3 |
+
|
| 4 |
+
pkill -f monitor.py || true
|
| 5 |
+
|
| 6 |
+
PROJECT_ROOT=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
|
| 7 |
+
cd "$PROJECT_ROOT"
|
| 8 |
+
export PYTHONPATH="$PROJECT_ROOT/src${PYTHONPATH:+:$PYTHONPATH}"
|
| 9 |
+
export CUDA_VISIBLE_DEVICES=${CUDA_VISIBLE_DEVICES:-0,1,2,3,4,5,6,7}
|
| 10 |
+
export PSI_OUTPUT_DIR=${PSI_OUTPUT_DIR:-$PROJECT_ROOT/runs}
|
| 11 |
+
export PSI_LOG_REPORT_TO=${PSI_LOG_REPORT_TO:-wandb}
|
| 12 |
+
|
| 13 |
+
source "$PROJECT_ROOT/scripts/activate_psi_env.sh"
|
| 14 |
+
[ -f .env ] && source .env
|
| 15 |
+
|
| 16 |
+
|
| 17 |
+
export OMP_NUM_THREADS=8
|
| 18 |
+
export TF_CPP_MIN_LOG_LEVEL=3
|
| 19 |
+
export PYTORCH_CUDA_ALLOC_CONF=${PYTORCH_CUDA_ALLOC_CONF:-expandable_segments:True}
|
| 20 |
+
export PSI_SKIP_VLM_MODEL=${PSI_SKIP_VLM_MODEL:-1}
|
| 21 |
+
|
| 22 |
+
# Keep JSON sources readable in this script, but pass compact JSON to tyro.
|
| 23 |
+
compact_json() {
|
| 24 |
+
python -c 'import json, sys; print(json.dumps(json.load(sys.stdin), separators=(",", ":")))'
|
| 25 |
+
}
|
| 26 |
+
|
| 27 |
+
PSI_SEED_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/Psi0/dataset/seed_all
|
| 28 |
+
PSI_SEED_METADATA=metadata/seed_metadata_v004.csv
|
| 29 |
+
PSI_TEXT_RECORDS_CACHE_DIR=${PSI_TEXT_RECORDS_CACHE_DIR:-$PSI_SEED_ROOT/.index_cache_multimodal_yth}
|
| 30 |
+
|
| 31 |
+
PSI_DANCE_MOTION_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/GR00T-WholeBodyControl/gear_sonic_deploy/reference/converted/psi0_dance/gvhmr
|
| 32 |
+
PSI_DANCE_AUDIO_VIDEO_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/dance_1723_video_psi0/dance_1723_video_10s
|
| 33 |
+
PSI_DANCE_VIDEO_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/dance_1723_video_psi0/dance_1723_video_10s
|
| 34 |
+
PSI_DANCE_VIDEO_EMBED_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/dance_1723_video_psi0/dance_1723_video_10s_embeds
|
| 35 |
+
PSI_VIDEO_DEMO_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/videogen/motion_example_0506_demo
|
| 36 |
+
PSI_2264_WITH_AUDIO_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/videogen/2264_with_audio
|
| 37 |
+
PSI_QWEN3_TTS_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/qwen3-tts
|
| 38 |
+
PSI_VIDEO_CLIP_DATASET_ROOT=${PSI_VIDEO_CLIP_DATASET_ROOT:-/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/videogen/psi0_video_clip_50f_dense_stride1_direct_full}
|
| 39 |
+
PSI_DANCE_VIDEO_CLIP_ROOT=$PSI_VIDEO_CLIP_DATASET_ROOT/dance_1723_video_10s
|
| 40 |
+
PSI_MOTION_EXAMPLE_VIDEO_CLIP_ROOT=$PSI_VIDEO_CLIP_DATASET_ROOT/motion_example_0506_demo
|
| 41 |
+
PSI_2264_WITH_AUDIO_VIDEO_CLIP_ROOT=$PSI_VIDEO_CLIP_DATASET_ROOT/2264_with_audio
|
| 42 |
+
PSI_DANCE_VIDEO_CLIP_EMBED_ROOT=${PSI_DANCE_VIDEO_CLIP_EMBED_ROOT:-$PSI_DANCE_VIDEO_CLIP_ROOT/embeds}
|
| 43 |
+
PSI_MOTION_EXAMPLE_VIDEO_CLIP_EMBED_ROOT=${PSI_MOTION_EXAMPLE_VIDEO_CLIP_EMBED_ROOT:-$PSI_MOTION_EXAMPLE_VIDEO_CLIP_ROOT/embeds}
|
| 44 |
+
PSI_2264_WITH_AUDIO_VIDEO_CLIP_EMBED_ROOT=${PSI_2264_WITH_AUDIO_VIDEO_CLIP_EMBED_ROOT:-$PSI_2264_WITH_AUDIO_VIDEO_CLIP_ROOT/embeds}
|
| 45 |
+
PSI_MOTION_EXAMPLE_VIDEO_EMBED_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/videogen/motion_example_0506_demo_video_embeds
|
| 46 |
+
PSI_AUDIO_RECORDS_CACHE_DIR=${PSI_AUDIO_RECORDS_CACHE_DIR:-$PSI_2264_WITH_AUDIO_ROOT/.index_cache_multimodal_yth}
|
| 47 |
+
PSI_AUDIO_COMMAND_RECORDS_CACHE_DIR=${PSI_AUDIO_COMMAND_RECORDS_CACHE_DIR:-$PSI_QWEN3_TTS_ROOT/.index_cache_multimodal_yth}
|
| 48 |
+
PSI_DANCE_AUDIO_NPY_ROOT=${PSI_DANCE_AUDIO_NPY_ROOT:-$PSI_DANCE_AUDIO_VIDEO_ROOT/audio_npy_yth}
|
| 49 |
+
PSI_DANCE_AUDIO_NPY_MANIFEST=${PSI_DANCE_AUDIO_NPY_MANIFEST:-$PSI_DANCE_AUDIO_VIDEO_ROOT/metadata_audio_npy.jsonl}
|
| 50 |
+
PSI_2264_AUDIO_NPY_ROOT=${PSI_2264_AUDIO_NPY_ROOT:-$PSI_2264_WITH_AUDIO_ROOT/audio_npy_yth}
|
| 51 |
+
PSI_2264_AUDIO_NPY_MANIFEST=${PSI_2264_AUDIO_NPY_MANIFEST:-$PSI_2264_WITH_AUDIO_ROOT/metadata_audio_npy.jsonl}
|
| 52 |
+
PSI_QWEN3_TTS_AUDIO_NPY_ROOT=${PSI_QWEN3_TTS_AUDIO_NPY_ROOT:-$PSI_QWEN3_TTS_ROOT/audio_npy_yth}
|
| 53 |
+
PSI_QWEN3_TTS_AUDIO_NPY_MANIFEST=${PSI_QWEN3_TTS_AUDIO_NPY_MANIFEST:-$PSI_QWEN3_TTS_ROOT/metadata_audio_npy.jsonl}
|
| 54 |
+
PSI_OMNI_HIDDEN_CACHE_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/videogen/psi0_omni_hidden_states_v23
|
| 55 |
+
|
| 56 |
+
if [ -f "$PSI_2264_AUDIO_NPY_MANIFEST" ]; then
|
| 57 |
+
PSI_2264_AUDIO_METADATA_JSONL=$PSI_2264_AUDIO_NPY_MANIFEST
|
| 58 |
+
PSI_2264_AUDIO_PATH_KEY=audio_path
|
| 59 |
+
else
|
| 60 |
+
PSI_2264_AUDIO_METADATA_JSONL=metadata_convert.jsonl
|
| 61 |
+
PSI_2264_AUDIO_PATH_KEY=video_path
|
| 62 |
+
fi
|
| 63 |
+
|
| 64 |
+
if [ -f "$PSI_DANCE_AUDIO_NPY_MANIFEST" ]; then
|
| 65 |
+
PSI_DANCE_AUDIO_SOURCE_JSON=$(cat <<JSON
|
| 66 |
+
{
|
| 67 |
+
"name": "psi0_dance_audio",
|
| 68 |
+
"dataset_root": "$PSI_DANCE_AUDIO_VIDEO_ROOT",
|
| 69 |
+
"metadata_jsonl": "$PSI_DANCE_AUDIO_NPY_MANIFEST",
|
| 70 |
+
"motion_dir_key": "converted_path",
|
| 71 |
+
"audio_path_key": "audio_path",
|
| 72 |
+
"records_cache_dir": "$PSI_AUDIO_RECORDS_CACHE_DIR",
|
| 73 |
+
"dataset_name": "seed-audio-g1"
|
| 74 |
+
}
|
| 75 |
+
JSON
|
| 76 |
+
)
|
| 77 |
+
else
|
| 78 |
+
PSI_DANCE_AUDIO_SOURCE_JSON=$(cat <<JSON
|
| 79 |
+
{
|
| 80 |
+
"name": "psi0_dance_audio",
|
| 81 |
+
"root_dir": "$PSI_DANCE_MOTION_ROOT",
|
| 82 |
+
"video_dir": "$PSI_DANCE_AUDIO_VIDEO_ROOT",
|
| 83 |
+
"records_cache_dir": "$PSI_AUDIO_RECORDS_CACHE_DIR",
|
| 84 |
+
"dataset_name": "seed-audio-g1"
|
| 85 |
+
}
|
| 86 |
+
JSON
|
| 87 |
+
)
|
| 88 |
+
fi
|
| 89 |
+
|
| 90 |
+
if [ -z "$PSI_AUDIO_SOURCES_JSON" ]; then
|
| 91 |
+
PSI_AUDIO_SOURCES_JSON=$(compact_json <<JSON
|
| 92 |
+
[
|
| 93 |
+
$PSI_DANCE_AUDIO_SOURCE_JSON,
|
| 94 |
+
{
|
| 95 |
+
"name": "2264_with_audio",
|
| 96 |
+
"dataset_root": "$PSI_2264_WITH_AUDIO_ROOT",
|
| 97 |
+
"metadata_jsonl": "$PSI_2264_AUDIO_METADATA_JSONL",
|
| 98 |
+
"motion_dir_key": "converted_path",
|
| 99 |
+
"audio_path_key": "$PSI_2264_AUDIO_PATH_KEY",
|
| 100 |
+
"records_cache_dir": "$PSI_AUDIO_RECORDS_CACHE_DIR",
|
| 101 |
+
"dataset_name": "2264-with-audio-g1"
|
| 102 |
+
}
|
| 103 |
+
]
|
| 104 |
+
JSON
|
| 105 |
+
)
|
| 106 |
+
fi
|
| 107 |
+
|
| 108 |
+
if [ -f "$PSI_QWEN3_TTS_AUDIO_NPY_MANIFEST" ]; then
|
| 109 |
+
PSI_AUDIO_COMMAND_SOURCE_JSON=$(cat <<JSON
|
| 110 |
+
{
|
| 111 |
+
"name": "qwen3_tts_seed_audio_command",
|
| 112 |
+
"root_dir": "$PSI_SEED_ROOT",
|
| 113 |
+
"dataset_root": "$PSI_QWEN3_TTS_ROOT",
|
| 114 |
+
"metadata_jsonl": "$PSI_QWEN3_TTS_AUDIO_NPY_MANIFEST",
|
| 115 |
+
"motion_dir_key": "filename",
|
| 116 |
+
"audio_path_key": "audio_path",
|
| 117 |
+
"instruction_key": "content_short_description",
|
| 118 |
+
"motion_metadata_csv": "$PSI_SEED_METADATA",
|
| 119 |
+
"motion_metadata_key": "filename",
|
| 120 |
+
"motion_metadata_path_key": "move_g1_path",
|
| 121 |
+
"audio_file_extensions": ".npy",
|
| 122 |
+
"records_cache_dir": "$PSI_AUDIO_COMMAND_RECORDS_CACHE_DIR",
|
| 123 |
+
"audio_task": "command",
|
| 124 |
+
"dataset_name": "seed-audio-command-g1"
|
| 125 |
+
}
|
| 126 |
+
JSON
|
| 127 |
+
)
|
| 128 |
+
else
|
| 129 |
+
PSI_AUDIO_COMMAND_SOURCE_JSON=$(cat <<JSON
|
| 130 |
+
{
|
| 131 |
+
"name": "qwen3_tts_seed_audio_command",
|
| 132 |
+
"root_dir": "$PSI_SEED_ROOT",
|
| 133 |
+
"dataset_root": "$PSI_QWEN3_TTS_ROOT",
|
| 134 |
+
"metadata_jsonl": "metadata.jsonl",
|
| 135 |
+
"motion_dir_key": "filename",
|
| 136 |
+
"audio_path_key": "audio_path",
|
| 137 |
+
"instruction_key": "content_short_description",
|
| 138 |
+
"motion_metadata_csv": "$PSI_SEED_METADATA",
|
| 139 |
+
"motion_metadata_key": "filename",
|
| 140 |
+
"motion_metadata_path_key": "move_g1_path",
|
| 141 |
+
"audio_file_extensions": ".wav",
|
| 142 |
+
"records_cache_dir": "$PSI_AUDIO_COMMAND_RECORDS_CACHE_DIR",
|
| 143 |
+
"audio_task": "command",
|
| 144 |
+
"dataset_name": "seed-audio-command-g1"
|
| 145 |
+
}
|
| 146 |
+
JSON
|
| 147 |
+
)
|
| 148 |
+
fi
|
| 149 |
+
|
| 150 |
+
if [ -z "$PSI_AUDIO_COMMAND_SOURCES_JSON" ]; then
|
| 151 |
+
PSI_AUDIO_COMMAND_SOURCES_JSON=$(compact_json <<JSON
|
| 152 |
+
[
|
| 153 |
+
$PSI_AUDIO_COMMAND_SOURCE_JSON
|
| 154 |
+
]
|
| 155 |
+
JSON
|
| 156 |
+
)
|
| 157 |
+
fi
|
| 158 |
+
|
| 159 |
+
if [ -z "$PSI_VIDEO_SOURCES_JSON" ]; then
|
| 160 |
+
PSI_VIDEO_SOURCES_JSON=$(compact_json <<JSON
|
| 161 |
+
[
|
| 162 |
+
{
|
| 163 |
+
"name": "psi0_dance_gvhmr",
|
| 164 |
+
"root_dir": "$PSI_DANCE_MOTION_ROOT",
|
| 165 |
+
"video_dir": "$PSI_DANCE_VIDEO_ROOT",
|
| 166 |
+
"video_embed_dir": "$PSI_DANCE_VIDEO_EMBED_ROOT",
|
| 167 |
+
"dataset_name": "seed-video-g1"
|
| 168 |
+
},
|
| 169 |
+
{
|
| 170 |
+
"name": "motion_example_0506_demo",
|
| 171 |
+
"dataset_root": "$PSI_VIDEO_DEMO_ROOT",
|
| 172 |
+
"metadata_jsonl": "metadata.jsonl",
|
| 173 |
+
"motion_dir_key": "converted_path",
|
| 174 |
+
"video_path_key": "video_path",
|
| 175 |
+
"instruction_key": "video_caption",
|
| 176 |
+
"video_embed_dir": "$PSI_MOTION_EXAMPLE_VIDEO_EMBED_ROOT",
|
| 177 |
+
"dataset_name": "motion-example-video-g1"
|
| 178 |
+
}
|
| 179 |
+
]
|
| 180 |
+
JSON
|
| 181 |
+
)
|
| 182 |
+
fi
|
| 183 |
+
|
| 184 |
+
if [ -z "$PSI_VIDEO_CLIP_SOURCES_JSON" ]; then
|
| 185 |
+
# Smoke/default cached training uses only video-clip sources whose hidden cache is complete.
|
| 186 |
+
# 2264_with_audio_clip50 is intentionally left out here until that cache group finishes.
|
| 187 |
+
PSI_VIDEO_CLIP_SOURCES_JSON=$(compact_json <<JSON
|
| 188 |
+
[
|
| 189 |
+
{
|
| 190 |
+
"name": "dance_1723_video_10s_clip50",
|
| 191 |
+
"dataset_root": "$PSI_DANCE_VIDEO_CLIP_ROOT",
|
| 192 |
+
"metadata_jsonl": "metadata.jsonl",
|
| 193 |
+
"motion_dir_key": "converted_path",
|
| 194 |
+
"video_path_key": "video_path",
|
| 195 |
+
"instruction_key": "video_caption",
|
| 196 |
+
"video_embed_dir": "$PSI_DANCE_VIDEO_CLIP_EMBED_ROOT",
|
| 197 |
+
"dataset_name": "dance-video-clip-g1"
|
| 198 |
+
},
|
| 199 |
+
{
|
| 200 |
+
"name": "motion_example_0506_demo_clip50",
|
| 201 |
+
"dataset_root": "$PSI_MOTION_EXAMPLE_VIDEO_CLIP_ROOT",
|
| 202 |
+
"metadata_jsonl": "metadata.jsonl",
|
| 203 |
+
"motion_dir_key": "converted_path",
|
| 204 |
+
"video_path_key": "video_path",
|
| 205 |
+
"instruction_key": "video_caption",
|
| 206 |
+
"video_embed_dir": "$PSI_MOTION_EXAMPLE_VIDEO_CLIP_EMBED_ROOT",
|
| 207 |
+
"dataset_name": "motion-example-video-clip-g1"
|
| 208 |
+
}
|
| 209 |
+
]
|
| 210 |
+
JSON
|
| 211 |
+
)
|
| 212 |
+
fi
|
| 213 |
+
|
| 214 |
+
# 7-task from-scratch mix, excluding legacy VIDEO_TEXT / VIDEO_ONLY.
|
| 215 |
+
# Smaller data sources get larger weights. These use a tempered inverse-size
|
| 216 |
+
# rule, (47M / source_size)^0.25, so rare modalities are upweighted without
|
| 217 |
+
# completely starving text/video_clip in BatchMixtureSampler.
|
| 218 |
+
# text stride1 frame samples: ~47,000,000
|
| 219 |
+
# music audio dance/audio frame samples use dense stride1 like video_clip
|
| 220 |
+
# audio_command wav files: 41,760
|
| 221 |
+
# dense video_clip metadata: 4,597,576
|
| 222 |
+
PSI_TEXT_TASK_WEIGHT=1
|
| 223 |
+
PSI_AUDIO_TEXT_TASK_WEIGHT=1
|
| 224 |
+
PSI_AUDIO_ONLY_TASK_WEIGHT=1
|
| 225 |
+
PSI_AUDIO_COMMAND_TEXT_TASK_WEIGHT=1
|
| 226 |
+
PSI_AUDIO_COMMAND_ONLY_TASK_WEIGHT=1
|
| 227 |
+
PSI_VIDEO_TEXT_TASK_WEIGHT=0
|
| 228 |
+
PSI_VIDEO_ONLY_TASK_WEIGHT=0
|
| 229 |
+
PSI_VIDEO_CLIP_TEXT_TASK_WEIGHT=1
|
| 230 |
+
PSI_VIDEO_CLIP_ONLY_TASK_WEIGHT=1
|
| 231 |
+
|
| 232 |
+
PSI_PRETRAIN_BACKBONE_PATH=/cpfs01/VideoGen/public_ckpt/Qwen/Qwen2.5-Omni-3B
|
| 233 |
+
NUM_PAST_FRAMES=0
|
| 234 |
+
STATE_HORIZON=$((NUM_PAST_FRAMES + 1))
|
| 235 |
+
ACTION_CHUNK_SIZE=30
|
| 236 |
+
ACTION_EXEC_HORIZON=20
|
| 237 |
+
RTC_PREFIX_FRAMES=$((ACTION_CHUNK_SIZE - ACTION_EXEC_HORIZON))
|
| 238 |
+
PROGRESS_DIM=5
|
| 239 |
+
MOTION_FPS=50
|
| 240 |
+
AUDIO_WINDOW_SECONDS=$(awk "BEGIN {printf \"%.6f\", ${ACTION_CHUNK_SIZE} / ${MOTION_FPS}}")
|
| 241 |
+
AUDIO_SAMPLE_STEP_FRAMES=1
|
| 242 |
+
VIDEO_SAMPLE_FPS=1.0
|
| 243 |
+
VIDEO_CLIP_SAMPLE_FPS=8.0
|
| 244 |
+
VIDEO_SIZE_PIXELS=50176
|
| 245 |
+
VIDEO_DECODER=ffmpeg
|
| 246 |
+
VIDEO_CACHE_SIZE=64
|
| 247 |
+
VIDEO_CLIP_MOTION_CACHE_SIZE=256
|
| 248 |
+
VIDEO_RECORD_BUILD_WORKERS=${VIDEO_RECORD_BUILD_WORKERS:-32}
|
| 249 |
+
VIDEO_RECORD_BUILD_CHUNK_SIZE=${VIDEO_RECORD_BUILD_CHUNK_SIZE:-4096}
|
| 250 |
+
VIDEO_CLIP_TRUST_METADATA=${VIDEO_CLIP_TRUST_METADATA:-1}
|
| 251 |
+
VIDEO_CLIP_RECORDS_CACHE_DIR=${VIDEO_CLIP_RECORDS_CACHE_DIR:-$PSI_VIDEO_CLIP_DATASET_ROOT/.index_cache_multimodal_yth}
|
| 252 |
+
if [ "$VIDEO_CLIP_TRUST_METADATA" = "1" ] || [ "$VIDEO_CLIP_TRUST_METADATA" = "true" ] || [ "$VIDEO_CLIP_TRUST_METADATA" = "True" ]; then
|
| 253 |
+
VIDEO_CLIP_TRUST_METADATA_FLAG="--data.video-clip.trust-metadata"
|
| 254 |
+
else
|
| 255 |
+
VIDEO_CLIP_TRUST_METADATA_FLAG="--data.video-clip.no-trust-metadata"
|
| 256 |
+
fi
|
| 257 |
+
|
| 258 |
+
# DATALOADER_NUM_WORKERS=4
|
| 259 |
+
# VAL_DATALOADER_NUM_WORKERS=2
|
| 260 |
+
|
| 261 |
+
# DATALOADER_NUM_WORKERS=${DATALOADER_NUM_WORKERS:-0}
|
| 262 |
+
# VAL_DATALOADER_NUM_WORKERS=${VAL_DATALOADER_NUM_WORKERS:-0}
|
| 263 |
+
|
| 264 |
+
DATALOADER_NUM_WORKERS=${DATALOADER_NUM_WORKERS:-2}
|
| 265 |
+
VAL_DATALOADER_NUM_WORKERS=${VAL_DATALOADER_NUM_WORKERS:-2}
|
| 266 |
+
TRAIN_BATCH_SIZE=${TRAIN_BATCH_SIZE:-64}
|
| 267 |
+
VAL_BATCH_SIZE=${VAL_BATCH_SIZE:-64}
|
| 268 |
+
GRADIENT_ACCUMULATION_STEPS=${GRADIENT_ACCUMULATION_STEPS:-1}
|
| 269 |
+
MAX_TRAINING_STEPS=${MAX_TRAINING_STEPS:-500000}
|
| 270 |
+
WARMUP_STEPS=${WARMUP_STEPS:-1000}
|
| 271 |
+
CHECKPOINTING_STEPS=${CHECKPOINTING_STEPS:-1000}
|
| 272 |
+
VALIDATION_STEPS=${VALIDATION_STEPS:-1000}
|
| 273 |
+
VAL_NUM_BATCHES=${VAL_NUM_BATCHES:-20}
|
| 274 |
+
MASTER_PORT=${MASTER_PORT:-29500}
|
| 275 |
+
|
| 276 |
+
NPROC_PER_NODE=$(echo $CUDA_VISIBLE_DEVICES | tr ',' '\n' | wc -l)
|
| 277 |
+
EFFECTIVE_TRAIN_BATCH_SIZE=$((TRAIN_BATCH_SIZE * NPROC_PER_NODE * GRADIENT_ACCUMULATION_STEPS))
|
| 278 |
+
|
| 279 |
+
echo "[cache-test-train] gpus=$CUDA_VISIBLE_DEVICES nproc=$NPROC_PER_NODE"
|
| 280 |
+
echo "Text root: $PSI_SEED_ROOT"
|
| 281 |
+
echo "Text records cache dir: $PSI_TEXT_RECORDS_CACHE_DIR"
|
| 282 |
+
echo "Dance motion root: $PSI_DANCE_MOTION_ROOT"
|
| 283 |
+
echo "Dance audio/video root: $PSI_DANCE_AUDIO_VIDEO_ROOT"
|
| 284 |
+
echo "Dance video root: $PSI_DANCE_VIDEO_ROOT"
|
| 285 |
+
echo "Dance video embed root: $PSI_DANCE_VIDEO_EMBED_ROOT"
|
| 286 |
+
echo "2264 with audio root: $PSI_2264_WITH_AUDIO_ROOT"
|
| 287 |
+
echo "Dance audio npy root: $PSI_DANCE_AUDIO_NPY_ROOT"
|
| 288 |
+
echo "Dance audio manifest: $PSI_DANCE_AUDIO_NPY_MANIFEST"
|
| 289 |
+
echo "2264 audio npy root: $PSI_2264_AUDIO_NPY_ROOT"
|
| 290 |
+
echo "2264 audio manifest: $PSI_2264_AUDIO_METADATA_JSONL"
|
| 291 |
+
echo "Audio records cache dir: $PSI_AUDIO_RECORDS_CACHE_DIR"
|
| 292 |
+
echo "Audio command records cache dir: $PSI_AUDIO_COMMAND_RECORDS_CACHE_DIR"
|
| 293 |
+
echo "Qwen3-TTS acdudio command root: $PSI_QWEN3_TTS_ROOT"
|
| 294 |
+
echo "Qwen3-TTS audio npy root: $PSI_QWEN3_TTS_AUDIO_NPY_ROOT"
|
| 295 |
+
echo "Qwen3-TTS audio manifest: $PSI_QWEN3_TTS_AUDIO_NPY_MANIFEST"
|
| 296 |
+
echo "Audio sources: $PSI_AUDIO_SOURCES_JSON"
|
| 297 |
+
echo "Audio command sources: $PSI_AUDIO_COMMAND_SOURCES_JSON"
|
| 298 |
+
echo "Video sources: $PSI_VIDEO_SOURCES_JSON"
|
| 299 |
+
echo "Video clip sources: $PSI_VIDEO_CLIP_SOURCES_JSON"
|
| 300 |
+
echo "DataLoader workers: train=$DATALOADER_NUM_WORKERS val=$VAL_DATALOADER_NUM_WORKERS"
|
| 301 |
+
echo "Sampler: task_mixture"
|
| 302 |
+
echo "Task weights: text=$PSI_TEXT_TASK_WEIGHT audio_text=$PSI_AUDIO_TEXT_TASK_WEIGHT audio_only=$PSI_AUDIO_ONLY_TASK_WEIGHT audio_command_text=$PSI_AUDIO_COMMAND_TEXT_TASK_WEIGHT audio_command_only=$PSI_AUDIO_COMMAND_ONLY_TASK_WEIGHT video_text=$PSI_VIDEO_TEXT_TASK_WEIGHT video_only=$PSI_VIDEO_ONLY_TASK_WEIGHT video_clip_text=$PSI_VIDEO_CLIP_TEXT_TASK_WEIGHT video_clip_only=$PSI_VIDEO_CLIP_ONLY_TASK_WEIGHT"
|
| 303 |
+
# echo "Resume run dir: $PSI_PRETRAIN_RUN_DIR"
|
| 304 |
+
echo "TextOp 39D action/state enabled"
|
| 305 |
+
echo "Future progress embedding: enabled, dim $PROGRESS_DIM"
|
| 306 |
+
echo "Backbone path: $PSI_PRETRAIN_BACKBONE_PATH"
|
| 307 |
+
echo "Output dir: $PSI_OUTPUT_DIR"
|
| 308 |
+
echo "Aligned audio window seconds: $AUDIO_WINDOW_SECONDS"
|
| 309 |
+
echo "Aligned audio sample step frames: $AUDIO_SAMPLE_STEP_FRAMES"
|
| 310 |
+
echo "Omni hidden cache root: $PSI_OMNI_HIDDEN_CACHE_ROOT"
|
| 311 |
+
echo "Video clip trust metadata: $VIDEO_CLIP_TRUST_METADATA"
|
| 312 |
+
echo "Video clip records cache dir: $VIDEO_CLIP_RECORDS_CACHE_DIR"
|
| 313 |
+
echo "Video record build workers: $VIDEO_RECORD_BUILD_WORKERS"
|
| 314 |
+
echo "Video record build chunk size: $VIDEO_RECORD_BUILD_CHUNK_SIZE"
|
| 315 |
+
echo "Micro batch: train=$TRAIN_BATCH_SIZE val=$VAL_BATCH_SIZE grad_accum=$GRADIENT_ACCUMULATION_STEPS effective_train_batch=$EFFECTIVE_TRAIN_BATCH_SIZE"
|
| 316 |
+
|
| 317 |
+
args="posttrain_v1_seed_text_audio_video_mix_psi0_config \
|
| 318 |
+
--seed=292285 \
|
| 319 |
+
--exp=posttrain \
|
| 320 |
+
--timestamp=$(date +"%y%m%d%H%M") \
|
| 321 |
+
--train.name=posttrain_v1_trimodal \
|
| 322 |
+
--train.data_parallel=ddp \
|
| 323 |
+
--train.mixed_precision=bf16 \
|
| 324 |
+
--train.train_batch_size=$TRAIN_BATCH_SIZE \
|
| 325 |
+
--train.val_batch_size=$VAL_BATCH_SIZE \
|
| 326 |
+
--train.num-workers=$DATALOADER_NUM_WORKERS \
|
| 327 |
+
--train.val-num-workers=$VAL_DATALOADER_NUM_WORKERS \
|
| 328 |
+
--train.resume_from_checkpoint=None \
|
| 329 |
+
--train.max_checkpoints_to_keep=1 \
|
| 330 |
+
--train.gradient_accumulation_steps=$GRADIENT_ACCUMULATION_STEPS \
|
| 331 |
+
--train.learning_rate=1e-4 \
|
| 332 |
+
--train.max_training_steps=$MAX_TRAINING_STEPS \
|
| 333 |
+
--train.warmup_ratio=None \
|
| 334 |
+
--train.warmup_steps=$WARMUP_STEPS \
|
| 335 |
+
--train.checkpointing_steps=$CHECKPOINTING_STEPS \
|
| 336 |
+
--train.validation_steps=$VALIDATION_STEPS \
|
| 337 |
+
--train.val_num_batches=$VAL_NUM_BATCHES \
|
| 338 |
+
--train.lr_scheduler_type=cosine_with_min_lr \
|
| 339 |
+
--train.lr_scheduler_kwargs.betas 0.9 0.999 \
|
| 340 |
+
--train.lr_scheduler_kwargs.weight_decay=0 \
|
| 341 |
+
--train.lr_scheduler_kwargs.eps=1e-8 \
|
| 342 |
+
--log.report_to=$PSI_LOG_REPORT_TO \
|
| 343 |
+
--train.output_dir=$PSI_OUTPUT_DIR \
|
| 344 |
+
--data.text.root-dir=$PSI_SEED_ROOT \
|
| 345 |
+
--data.text.metadata-csv=$PSI_SEED_METADATA \
|
| 346 |
+
--data.text.records-cache-dir=$PSI_TEXT_RECORDS_CACHE_DIR \
|
| 347 |
+
--data.text.use-omni-hidden-cache \
|
| 348 |
+
--data.text.require-omni-hidden-cache \
|
| 349 |
+
--data.text.omni-hidden-root=$PSI_OMNI_HIDDEN_CACHE_ROOT \
|
| 350 |
+
--data.text-ratio=$PSI_TEXT_TASK_WEIGHT \
|
| 351 |
+
--data.audio.sources-json=$PSI_AUDIO_SOURCES_JSON \
|
| 352 |
+
--data.audio-ratio=$PSI_AUDIO_TEXT_TASK_WEIGHT \
|
| 353 |
+
--data.audio-only-ratio=$PSI_AUDIO_ONLY_TASK_WEIGHT \
|
| 354 |
+
--data.audio.sample-stride=1 \
|
| 355 |
+
--data.audio.sample-step-frames=$AUDIO_SAMPLE_STEP_FRAMES \
|
| 356 |
+
--data.audio.motion-fps=$MOTION_FPS \
|
| 357 |
+
--data.audio.audio-sampling-rate=16000 \
|
| 358 |
+
--data.audio.audio-window-seconds=$AUDIO_WINDOW_SECONDS \
|
| 359 |
+
--data.audio.audio-left-context-seconds=0.0 \
|
| 360 |
+
--data.audio.use-omni-hidden-cache \
|
| 361 |
+
--data.audio.require-omni-hidden-cache \
|
| 362 |
+
--data.audio.omni-hidden-root=$PSI_OMNI_HIDDEN_CACHE_ROOT \
|
| 363 |
+
--data.audio-command.sources-json=$PSI_AUDIO_COMMAND_SOURCES_JSON \
|
| 364 |
+
--data.audio-command-ratio=$PSI_AUDIO_COMMAND_TEXT_TASK_WEIGHT \
|
| 365 |
+
--data.audio-command-only-ratio=$PSI_AUDIO_COMMAND_ONLY_TASK_WEIGHT \
|
| 366 |
+
--data.audio-command.sample-stride=1 \
|
| 367 |
+
--data.audio-command.sample-step-frames=$AUDIO_SAMPLE_STEP_FRAMES \
|
| 368 |
+
--data.audio-command.motion-fps=$MOTION_FPS \
|
| 369 |
+
--data.audio-command.audio-sampling-rate=16000 \
|
| 370 |
+
--data.audio-command.audio-window-seconds=$AUDIO_WINDOW_SECONDS \
|
| 371 |
+
--data.audio-command.audio-left-context-seconds=0.0 \
|
| 372 |
+
--data.audio-command.full-audio \
|
| 373 |
+
--data.audio-command.use-omni-hidden-cache \
|
| 374 |
+
--data.audio-command.require-omni-hidden-cache \
|
| 375 |
+
--data.audio-command.omni-hidden-root=$PSI_OMNI_HIDDEN_CACHE_ROOT \
|
| 376 |
+
--data.video-ratio=$PSI_VIDEO_TEXT_TASK_WEIGHT \
|
| 377 |
+
--data.video-only-ratio=$PSI_VIDEO_ONLY_TASK_WEIGHT \
|
| 378 |
+
--data.video.sample-stride=1 \
|
| 379 |
+
--data.video.motion-fps=$MOTION_FPS \
|
| 380 |
+
--data.video.video-sample-fps=$VIDEO_SAMPLE_FPS \
|
| 381 |
+
--data.video.video-size-pixels=$VIDEO_SIZE_PIXELS \
|
| 382 |
+
--data.video.video-decoder=$VIDEO_DECODER \
|
| 383 |
+
--data.video.video-cache-size=$VIDEO_CACHE_SIZE \
|
| 384 |
+
--data.video.record-build-workers=$VIDEO_RECORD_BUILD_WORKERS \
|
| 385 |
+
--data.video.record-build-chunk-size=$VIDEO_RECORD_BUILD_CHUNK_SIZE \
|
| 386 |
+
--data.video.sources-json=$PSI_VIDEO_SOURCES_JSON \
|
| 387 |
+
--data.video-clip-text-ratio=$PSI_VIDEO_CLIP_TEXT_TASK_WEIGHT \
|
| 388 |
+
--data.video-clip-only-ratio=$PSI_VIDEO_CLIP_ONLY_TASK_WEIGHT \
|
| 389 |
+
--data.video-clip.sources-json=$PSI_VIDEO_CLIP_SOURCES_JSON \
|
| 390 |
+
--data.video-clip.sample-stride=1 \
|
| 391 |
+
--data.video-clip.motion-fps=$MOTION_FPS \
|
| 392 |
+
--data.video-clip.video-sample-fps=$VIDEO_CLIP_SAMPLE_FPS \
|
| 393 |
+
--data.video-clip.video-size-pixels=$VIDEO_SIZE_PIXELS \
|
| 394 |
+
--data.video-clip.video-decoder=$VIDEO_DECODER \
|
| 395 |
+
--data.video-clip.video-cache-size=$VIDEO_CACHE_SIZE \
|
| 396 |
+
--data.video-clip.motion-cache-size=$VIDEO_CLIP_MOTION_CACHE_SIZE \
|
| 397 |
+
--data.video-clip.record-build-workers=$VIDEO_RECORD_BUILD_WORKERS \
|
| 398 |
+
--data.video-clip.record-build-chunk-size=$VIDEO_RECORD_BUILD_CHUNK_SIZE \
|
| 399 |
+
$VIDEO_CLIP_TRUST_METADATA_FLAG \
|
| 400 |
+
--data.video-clip.records-cache-dir=$VIDEO_CLIP_RECORDS_CACHE_DIR \
|
| 401 |
+
--data.video-clip.use-omni-hidden-cache \
|
| 402 |
+
--data.video-clip.require-omni-hidden-cache \
|
| 403 |
+
--data.video-clip.omni-hidden-root=$PSI_OMNI_HIDDEN_CACHE_ROOT \
|
| 404 |
+
--data.video-clip.action-exec-horizon=$ACTION_EXEC_HORIZON \
|
| 405 |
+
--data.sampler=task_mixture \
|
| 406 |
+
--data.val-task=text,audio,audio_only,audio_command,audio_command_only,video_clip,video_clip_only \
|
| 407 |
+
--data.transform.repack.num-past-frames=$NUM_PAST_FRAMES \
|
| 408 |
+
--data.transform.repack.textop-actions \
|
| 409 |
+
--data.transform.repack.action-chunk-size=$ACTION_CHUNK_SIZE \
|
| 410 |
+
--data.transform.repack.rtc-prefix-frames=$RTC_PREFIX_FRAMES \
|
| 411 |
+
--data.transform.repack.pad-action-dim=39 \
|
| 412 |
+
--data.transform.repack.pad-state-dim=39 \
|
| 413 |
+
--data.transform.field.stat-path=assets/stats/seed_all_g1_motion39_textop_stats.json \
|
| 414 |
+
--data.transform.field.normalize-state \
|
| 415 |
+
--data.transform.field.pad-action-dim=39 \
|
| 416 |
+
--data.transform.field.pad-state-dim=39 \
|
| 417 |
+
--data.transform.model.resize.size 240 320 \
|
| 418 |
+
--data.transform.model.center_crop.size 240 320 \
|
| 419 |
+
--data.transform.model.no-img-aug \
|
| 420 |
+
--data.transform.model.text-only \
|
| 421 |
+
--data.transform.model.audio-only \
|
| 422 |
+
--data.transform.model.video-only \
|
| 423 |
+
--data.transform.model.video-size-pixels=$VIDEO_SIZE_PIXELS \
|
| 424 |
+
--model.model_name_or_path=$PSI_PRETRAIN_BACKBONE_PATH \
|
| 425 |
+
--model.noise-scheduler=flow \
|
| 426 |
+
--model.n_conditions=0 \
|
| 427 |
+
--model.action-chunk-size=$ACTION_CHUNK_SIZE \
|
| 428 |
+
--model.action-dim=39 \
|
| 429 |
+
--model.action-exec-horizon=$ACTION_EXEC_HORIZON \
|
| 430 |
+
--model.observation-horizon=$STATE_HORIZON \
|
| 431 |
+
--model.odim=39 \
|
| 432 |
+
--model.view_feature_dim=2048 \
|
| 433 |
+
--model.use-future-progress-embedding \
|
| 434 |
+
--model.future-progress-dim=$PROGRESS_DIM \
|
| 435 |
+
--model.no-use_film \
|
| 436 |
+
--model.no-combined_temb \
|
| 437 |
+
--model.rtc \
|
| 438 |
+
--model.max-delay=11 \
|
| 439 |
+
--train.max-grad-value=1 \
|
| 440 |
+
--train.loss-group-weight.joint-abs=1.0 \
|
| 441 |
+
--train.loss-group-weight.root-delta-xy=3.0 \
|
| 442 |
+
--train.loss-group-weight.root-rot5-delta=10.0 \
|
| 443 |
+
--train.loss-group-weight.root-height=15.0 \
|
| 444 |
+
--train.loss-group-weight.foot-contact=0.1
|
| 445 |
+
"
|
| 446 |
+
torchrun --nproc_per_node=$NPROC_PER_NODE --master_port=$MASTER_PORT scripts/train.py \
|
| 447 |
+
$args "$@"
|
prebuild-multimodal-records-cache.sh
ADDED
|
@@ -0,0 +1,306 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#!/usr/bin/env bash
|
| 2 |
+
set -eo pipefail
|
| 3 |
+
|
| 4 |
+
PROJECT_ROOT=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
|
| 5 |
+
cd "$PROJECT_ROOT"
|
| 6 |
+
export PYTHONPATH="$PROJECT_ROOT/src${PYTHONPATH:+:$PYTHONPATH}"
|
| 7 |
+
|
| 8 |
+
source "$PROJECT_ROOT/scripts/activate_psi_env.sh"
|
| 9 |
+
[ -f .env ] && source .env
|
| 10 |
+
|
| 11 |
+
export OMP_NUM_THREADS=${OMP_NUM_THREADS:-8}
|
| 12 |
+
export TF_CPP_MIN_LOG_LEVEL=${TF_CPP_MIN_LOG_LEVEL:-3}
|
| 13 |
+
export PSI_VIDEO_UNIQUE_RECORD_CACHE_FALLBACK=0
|
| 14 |
+
export PSI_REQUIRE_PREBUILT_VIDEO_RECORD_CACHE=0
|
| 15 |
+
export PSI_VIDEO_RECORD_PATH_REWRITE_FROM=${PSI_VIDEO_RECORD_PATH_REWRITE_FROM:-/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/videogen/omni_hidden_states_v23_oss}
|
| 16 |
+
export PSI_VIDEO_RECORD_PATH_REWRITE_TO=${PSI_VIDEO_RECORD_PATH_REWRITE_TO:-/mnt/data/data/omni_hidden_states_v23_oss}
|
| 17 |
+
|
| 18 |
+
compact_json() {
|
| 19 |
+
python -c 'import json, sys; print(json.dumps(json.load(sys.stdin), separators=(",", ":")))'
|
| 20 |
+
}
|
| 21 |
+
|
| 22 |
+
PSI_SEED_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/Psi0/dataset/seed_all
|
| 23 |
+
PSI_SEED_METADATA=metadata/seed_metadata_v004.csv
|
| 24 |
+
PSI_OMNI_OSS_ROOT=${PSI_OMNI_HIDDEN_CACHE_ROOT:-/mnt/data/data/omni_hidden_states_v23_oss}
|
| 25 |
+
PSI_OMNI_OSS_TRAINING_SUPPORT_ROOT=${PSI_OMNI_OSS_TRAINING_SUPPORT_ROOT:-$PSI_OMNI_OSS_ROOT/training_support}
|
| 26 |
+
PSI_SOURCE_SPLIT_ROOT=${PSI_SOURCE_SPLIT_ROOT:-$PSI_OMNI_OSS_TRAINING_SUPPORT_ROOT/source_split_v1}
|
| 27 |
+
export PSI_OMNI_OSS_RUNTIME_INDEX_DIR=${PSI_OMNI_OSS_RUNTIME_INDEX_DIR:-$PSI_SOURCE_SPLIT_ROOT/runtime_indexes}
|
| 28 |
+
PSI_TEXT_RECORDS_CACHE_DIR=${PSI_TEXT_RECORDS_CACHE_DIR:-$PSI_SOURCE_SPLIT_ROOT/text_records}
|
| 29 |
+
|
| 30 |
+
PSI_DANCE_MOTION_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/GR00T-WholeBodyControl/gear_sonic_deploy/reference/converted/psi0_dance/gvhmr
|
| 31 |
+
PSI_DANCE_AUDIO_VIDEO_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/dance_1723_video_psi0/dance_1723_video_10s
|
| 32 |
+
PSI_DANCE_VIDEO_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/dance_1723_video_psi0/dance_1723_video_10s
|
| 33 |
+
PSI_DANCE_VIDEO_EMBED_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/dance_1723_video_psi0/dance_1723_video_10s_embeds
|
| 34 |
+
PSI_VIDEO_DEMO_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/videogen/motion_example_0506_demo
|
| 35 |
+
PSI_2264_WITH_AUDIO_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/videogen/2264_with_audio
|
| 36 |
+
PSI_QWEN3_TTS_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/qwen3-tts
|
| 37 |
+
PSI_VIDEO_CLIP_DATASET_ROOT=${PSI_VIDEO_CLIP_DATASET_ROOT:-$PSI_OMNI_OSS_TRAINING_SUPPORT_ROOT/video_clip_dense_stride1}
|
| 38 |
+
PSI_DANCE_VIDEO_CLIP_ROOT=$PSI_VIDEO_CLIP_DATASET_ROOT/dance_1723_video_10s
|
| 39 |
+
PSI_MOTION_EXAMPLE_VIDEO_CLIP_ROOT=$PSI_VIDEO_CLIP_DATASET_ROOT/motion_example_0506_demo
|
| 40 |
+
PSI_2264_WITH_AUDIO_VIDEO_CLIP_ROOT=$PSI_VIDEO_CLIP_DATASET_ROOT/2264_with_audio
|
| 41 |
+
PSI_DANCE_VIDEO_CLIP_EMBED_ROOT=${PSI_DANCE_VIDEO_CLIP_EMBED_ROOT:-$PSI_DANCE_VIDEO_CLIP_ROOT/embeds}
|
| 42 |
+
PSI_MOTION_EXAMPLE_VIDEO_CLIP_EMBED_ROOT=${PSI_MOTION_EXAMPLE_VIDEO_CLIP_EMBED_ROOT:-$PSI_MOTION_EXAMPLE_VIDEO_CLIP_ROOT/embeds}
|
| 43 |
+
PSI_2264_WITH_AUDIO_VIDEO_CLIP_EMBED_ROOT=${PSI_2264_WITH_AUDIO_VIDEO_CLIP_EMBED_ROOT:-$PSI_2264_WITH_AUDIO_VIDEO_CLIP_ROOT/embeds}
|
| 44 |
+
PSI_MOTION_EXAMPLE_VIDEO_EMBED_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/videogen/motion_example_0506_demo_video_embeds
|
| 45 |
+
PSI_AUDIO_RECORDS_CACHE_DIR=${PSI_AUDIO_RECORDS_CACHE_DIR:-$PSI_SOURCE_SPLIT_ROOT/audio_records}
|
| 46 |
+
PSI_AUDIO_COMMAND_RECORDS_CACHE_DIR=${PSI_AUDIO_COMMAND_RECORDS_CACHE_DIR:-$PSI_SOURCE_SPLIT_ROOT/audio_command_records}
|
| 47 |
+
PSI_DANCE_AUDIO_NPY_ROOT=${PSI_DANCE_AUDIO_NPY_ROOT:-$PSI_DANCE_AUDIO_VIDEO_ROOT/audio_npy_yth}
|
| 48 |
+
PSI_DANCE_AUDIO_NPY_MANIFEST=${PSI_DANCE_AUDIO_NPY_MANIFEST:-$PSI_DANCE_AUDIO_VIDEO_ROOT/metadata_audio_npy.jsonl}
|
| 49 |
+
PSI_2264_AUDIO_NPY_MANIFEST=${PSI_2264_AUDIO_NPY_MANIFEST:-$PSI_2264_WITH_AUDIO_ROOT/metadata_audio_npy.jsonl}
|
| 50 |
+
PSI_QWEN3_TTS_AUDIO_NPY_ROOT=${PSI_QWEN3_TTS_AUDIO_NPY_ROOT:-$PSI_QWEN3_TTS_ROOT/audio_npy_yth}
|
| 51 |
+
PSI_QWEN3_TTS_AUDIO_NPY_MANIFEST=${PSI_QWEN3_TTS_AUDIO_NPY_MANIFEST:-$PSI_QWEN3_TTS_ROOT/metadata_audio_npy.jsonl}
|
| 52 |
+
|
| 53 |
+
if [ -f "$PSI_2264_AUDIO_NPY_MANIFEST" ]; then
|
| 54 |
+
PSI_2264_AUDIO_METADATA_JSONL=$PSI_2264_AUDIO_NPY_MANIFEST
|
| 55 |
+
PSI_2264_AUDIO_PATH_KEY=audio_path
|
| 56 |
+
else
|
| 57 |
+
PSI_2264_AUDIO_METADATA_JSONL=metadata_convert.jsonl
|
| 58 |
+
PSI_2264_AUDIO_PATH_KEY=video_path
|
| 59 |
+
fi
|
| 60 |
+
|
| 61 |
+
if [ -f "$PSI_DANCE_AUDIO_NPY_MANIFEST" ]; then
|
| 62 |
+
PSI_DANCE_AUDIO_SOURCE_JSON=$(cat <<JSON
|
| 63 |
+
{
|
| 64 |
+
"name": "psi0_dance_audio",
|
| 65 |
+
"split_source_namespace": "dance_1723_video_10s",
|
| 66 |
+
"dataset_root": "$PSI_DANCE_AUDIO_VIDEO_ROOT",
|
| 67 |
+
"metadata_jsonl": "$PSI_DANCE_AUDIO_NPY_MANIFEST",
|
| 68 |
+
"motion_dir_key": "converted_path",
|
| 69 |
+
"audio_path_key": "audio_path",
|
| 70 |
+
"records_cache_dir": "$PSI_AUDIO_RECORDS_CACHE_DIR",
|
| 71 |
+
"dataset_name": "seed-audio-g1"
|
| 72 |
+
}
|
| 73 |
+
JSON
|
| 74 |
+
)
|
| 75 |
+
else
|
| 76 |
+
PSI_DANCE_AUDIO_SOURCE_JSON=$(cat <<JSON
|
| 77 |
+
{
|
| 78 |
+
"name": "psi0_dance_audio",
|
| 79 |
+
"split_source_namespace": "dance_1723_video_10s",
|
| 80 |
+
"root_dir": "$PSI_DANCE_MOTION_ROOT",
|
| 81 |
+
"video_dir": "$PSI_DANCE_AUDIO_VIDEO_ROOT",
|
| 82 |
+
"records_cache_dir": "$PSI_AUDIO_RECORDS_CACHE_DIR",
|
| 83 |
+
"dataset_name": "seed-audio-g1"
|
| 84 |
+
}
|
| 85 |
+
JSON
|
| 86 |
+
)
|
| 87 |
+
fi
|
| 88 |
+
|
| 89 |
+
if [ -z "$PSI_AUDIO_SOURCES_JSON" ]; then
|
| 90 |
+
PSI_AUDIO_SOURCES_JSON=$(compact_json <<JSON
|
| 91 |
+
[
|
| 92 |
+
$PSI_DANCE_AUDIO_SOURCE_JSON,
|
| 93 |
+
{
|
| 94 |
+
"name": "2264_with_audio",
|
| 95 |
+
"split_source_namespace": "2264_with_audio",
|
| 96 |
+
"dataset_root": "$PSI_2264_WITH_AUDIO_ROOT",
|
| 97 |
+
"metadata_jsonl": "$PSI_2264_AUDIO_METADATA_JSONL",
|
| 98 |
+
"motion_dir_key": "converted_path",
|
| 99 |
+
"audio_path_key": "$PSI_2264_AUDIO_PATH_KEY",
|
| 100 |
+
"records_cache_dir": "$PSI_AUDIO_RECORDS_CACHE_DIR",
|
| 101 |
+
"dataset_name": "2264-with-audio-g1"
|
| 102 |
+
}
|
| 103 |
+
]
|
| 104 |
+
JSON
|
| 105 |
+
)
|
| 106 |
+
fi
|
| 107 |
+
|
| 108 |
+
if [ -f "$PSI_QWEN3_TTS_AUDIO_NPY_MANIFEST" ]; then
|
| 109 |
+
PSI_AUDIO_COMMAND_SOURCE_JSON=$(cat <<JSON
|
| 110 |
+
{
|
| 111 |
+
"name": "qwen3_tts_seed_audio_command",
|
| 112 |
+
"split_source_namespace": "seed",
|
| 113 |
+
"root_dir": "$PSI_SEED_ROOT",
|
| 114 |
+
"dataset_root": "$PSI_QWEN3_TTS_ROOT",
|
| 115 |
+
"metadata_jsonl": "$PSI_QWEN3_TTS_AUDIO_NPY_MANIFEST",
|
| 116 |
+
"motion_dir_key": "filename",
|
| 117 |
+
"audio_path_key": "audio_path",
|
| 118 |
+
"instruction_key": "content_short_description",
|
| 119 |
+
"motion_metadata_csv": "$PSI_SEED_METADATA",
|
| 120 |
+
"motion_metadata_key": "filename",
|
| 121 |
+
"motion_metadata_path_key": "move_g1_path",
|
| 122 |
+
"audio_file_extensions": ".npy",
|
| 123 |
+
"records_cache_dir": "$PSI_AUDIO_COMMAND_RECORDS_CACHE_DIR",
|
| 124 |
+
"audio_task": "command",
|
| 125 |
+
"dataset_name": "seed-audio-command-g1"
|
| 126 |
+
}
|
| 127 |
+
JSON
|
| 128 |
+
)
|
| 129 |
+
else
|
| 130 |
+
PSI_AUDIO_COMMAND_SOURCE_JSON=$(cat <<JSON
|
| 131 |
+
{
|
| 132 |
+
"name": "qwen3_tts_seed_audio_command",
|
| 133 |
+
"split_source_namespace": "seed",
|
| 134 |
+
"root_dir": "$PSI_SEED_ROOT",
|
| 135 |
+
"dataset_root": "$PSI_QWEN3_TTS_ROOT",
|
| 136 |
+
"metadata_jsonl": "metadata.jsonl",
|
| 137 |
+
"motion_dir_key": "filename",
|
| 138 |
+
"audio_path_key": "audio_path",
|
| 139 |
+
"instruction_key": "content_short_description",
|
| 140 |
+
"motion_metadata_csv": "$PSI_SEED_METADATA",
|
| 141 |
+
"motion_metadata_key": "filename",
|
| 142 |
+
"motion_metadata_path_key": "move_g1_path",
|
| 143 |
+
"audio_file_extensions": ".wav",
|
| 144 |
+
"records_cache_dir": "$PSI_AUDIO_COMMAND_RECORDS_CACHE_DIR",
|
| 145 |
+
"audio_task": "command",
|
| 146 |
+
"dataset_name": "seed-audio-command-g1"
|
| 147 |
+
}
|
| 148 |
+
JSON
|
| 149 |
+
)
|
| 150 |
+
fi
|
| 151 |
+
|
| 152 |
+
if [ -z "$PSI_AUDIO_COMMAND_SOURCES_JSON" ]; then
|
| 153 |
+
PSI_AUDIO_COMMAND_SOURCES_JSON=$(compact_json <<JSON
|
| 154 |
+
[
|
| 155 |
+
$PSI_AUDIO_COMMAND_SOURCE_JSON
|
| 156 |
+
]
|
| 157 |
+
JSON
|
| 158 |
+
)
|
| 159 |
+
fi
|
| 160 |
+
|
| 161 |
+
if [ -z "$PSI_VIDEO_SOURCES_JSON" ]; then
|
| 162 |
+
PSI_VIDEO_SOURCES_JSON=$(compact_json <<JSON
|
| 163 |
+
[
|
| 164 |
+
{
|
| 165 |
+
"name": "psi0_dance_gvhmr",
|
| 166 |
+
"root_dir": "$PSI_DANCE_MOTION_ROOT",
|
| 167 |
+
"video_dir": "$PSI_DANCE_VIDEO_ROOT",
|
| 168 |
+
"video_embed_dir": "$PSI_DANCE_VIDEO_EMBED_ROOT",
|
| 169 |
+
"dataset_name": "seed-video-g1"
|
| 170 |
+
},
|
| 171 |
+
{
|
| 172 |
+
"name": "motion_example_0506_demo",
|
| 173 |
+
"dataset_root": "$PSI_VIDEO_DEMO_ROOT",
|
| 174 |
+
"metadata_jsonl": "metadata.jsonl",
|
| 175 |
+
"motion_dir_key": "converted_path",
|
| 176 |
+
"video_path_key": "video_path",
|
| 177 |
+
"instruction_key": "video_caption",
|
| 178 |
+
"video_embed_dir": "$PSI_MOTION_EXAMPLE_VIDEO_EMBED_ROOT",
|
| 179 |
+
"dataset_name": "motion-example-video-g1"
|
| 180 |
+
}
|
| 181 |
+
]
|
| 182 |
+
JSON
|
| 183 |
+
)
|
| 184 |
+
fi
|
| 185 |
+
|
| 186 |
+
if [ -z "$PSI_VIDEO_CLIP_SOURCES_JSON" ]; then
|
| 187 |
+
PSI_VIDEO_CLIP_SOURCES_JSON=$(compact_json <<JSON
|
| 188 |
+
[
|
| 189 |
+
{
|
| 190 |
+
"name": "dance_1723_video_10s_clip50",
|
| 191 |
+
"split_source_namespace": "dance_1723_video_10s",
|
| 192 |
+
"dataset_root": "$PSI_DANCE_VIDEO_CLIP_ROOT",
|
| 193 |
+
"metadata_jsonl": "metadata.jsonl",
|
| 194 |
+
"motion_dir_key": "converted_path",
|
| 195 |
+
"video_path_key": "video_path",
|
| 196 |
+
"instruction_key": "video_caption",
|
| 197 |
+
"video_embed_dir": "$PSI_DANCE_VIDEO_CLIP_EMBED_ROOT",
|
| 198 |
+
"dataset_name": "dance-video-clip-g1"
|
| 199 |
+
},
|
| 200 |
+
{
|
| 201 |
+
"name": "motion_example_0506_demo_clip50",
|
| 202 |
+
"split_source_namespace": "motion_example_0506_demo",
|
| 203 |
+
"dataset_root": "$PSI_MOTION_EXAMPLE_VIDEO_CLIP_ROOT",
|
| 204 |
+
"metadata_jsonl": "metadata.jsonl",
|
| 205 |
+
"motion_dir_key": "converted_path",
|
| 206 |
+
"video_path_key": "video_path",
|
| 207 |
+
"instruction_key": "video_caption",
|
| 208 |
+
"video_embed_dir": "$PSI_MOTION_EXAMPLE_VIDEO_CLIP_EMBED_ROOT",
|
| 209 |
+
"dataset_name": "motion-example-video-clip-g1"
|
| 210 |
+
},
|
| 211 |
+
{
|
| 212 |
+
"name": "2264_with_audio_clip50",
|
| 213 |
+
"split_source_namespace": "2264_with_audio",
|
| 214 |
+
"dataset_root": "$PSI_2264_WITH_AUDIO_VIDEO_CLIP_ROOT",
|
| 215 |
+
"metadata_jsonl": "metadata.jsonl",
|
| 216 |
+
"motion_dir_key": "converted_path",
|
| 217 |
+
"video_path_key": "video_path",
|
| 218 |
+
"instruction_key": "video_caption",
|
| 219 |
+
"video_embed_dir": "$PSI_2264_WITH_AUDIO_VIDEO_CLIP_EMBED_ROOT",
|
| 220 |
+
"dataset_name": "2264-with-audio-video-clip-g1"
|
| 221 |
+
}
|
| 222 |
+
]
|
| 223 |
+
JSON
|
| 224 |
+
)
|
| 225 |
+
fi
|
| 226 |
+
|
| 227 |
+
ACTION_CHUNK_SIZE=${ACTION_CHUNK_SIZE:-30}
|
| 228 |
+
ACTION_EXEC_HORIZON=${ACTION_EXEC_HORIZON:-20}
|
| 229 |
+
NUM_PAST_FRAMES=${NUM_PAST_FRAMES:-0}
|
| 230 |
+
VIDEO_SAMPLE_FPS=${VIDEO_SAMPLE_FPS:-1.0}
|
| 231 |
+
VIDEO_CLIP_SAMPLE_FPS=${VIDEO_CLIP_SAMPLE_FPS:-8.0}
|
| 232 |
+
VIDEO_SIZE_PIXELS=${VIDEO_SIZE_PIXELS:-50176}
|
| 233 |
+
VIDEO_DECODER=${VIDEO_DECODER:-ffmpeg}
|
| 234 |
+
VIDEO_RECORD_BUILD_WORKERS=${VIDEO_RECORD_BUILD_WORKERS:-32}
|
| 235 |
+
VIDEO_RECORD_BUILD_CHUNK_SIZE=${VIDEO_RECORD_BUILD_CHUNK_SIZE:-4096}
|
| 236 |
+
VIDEO_CLIP_TRUST_METADATA=${VIDEO_CLIP_TRUST_METADATA:-1}
|
| 237 |
+
VIDEO_CLIP_RECORDS_CACHE_DIR=${VIDEO_CLIP_RECORDS_CACHE_DIR:-$PSI_SOURCE_SPLIT_ROOT/video_clip_records}
|
| 238 |
+
PREBUILD_TASKS=${PREBUILD_TASKS:-text,audio,audio_command,video_clip}
|
| 239 |
+
PREBUILD_SPLITS=${PREBUILD_SPLITS:-train,val}
|
| 240 |
+
|
| 241 |
+
if [ "$VIDEO_CLIP_TRUST_METADATA" = "1" ] || [ "$VIDEO_CLIP_TRUST_METADATA" = "true" ] || [ "$VIDEO_CLIP_TRUST_METADATA" = "True" ]; then
|
| 242 |
+
PREBUILD_VIDEO_CLIP_TRUST_METADATA_FLAG="--video-clip-trust-metadata"
|
| 243 |
+
else
|
| 244 |
+
PREBUILD_VIDEO_CLIP_TRUST_METADATA_FLAG="--no-video-clip-trust-metadata"
|
| 245 |
+
fi
|
| 246 |
+
|
| 247 |
+
echo "Prebuild tasks: $PREBUILD_TASKS"
|
| 248 |
+
echo "Prebuild splits: $PREBUILD_SPLITS"
|
| 249 |
+
echo "Source split OSS package root: $PSI_SOURCE_SPLIT_ROOT"
|
| 250 |
+
echo "Runtime index package dir: $PSI_OMNI_OSS_RUNTIME_INDEX_DIR"
|
| 251 |
+
echo "Text cache dir: $PSI_TEXT_RECORDS_CACHE_DIR"
|
| 252 |
+
echo "Audio cache dir: $PSI_AUDIO_RECORDS_CACHE_DIR"
|
| 253 |
+
echo "Audio command cache dir: $PSI_AUDIO_COMMAND_RECORDS_CACHE_DIR"
|
| 254 |
+
echo "Dance audio manifest: $PSI_DANCE_AUDIO_NPY_MANIFEST"
|
| 255 |
+
echo "2264 audio manifest: $PSI_2264_AUDIO_METADATA_JSONL"
|
| 256 |
+
echo "Qwen3-TTS audio manifest: $PSI_QWEN3_TTS_AUDIO_NPY_MANIFEST"
|
| 257 |
+
echo "Video clip cache dir: $VIDEO_CLIP_RECORDS_CACHE_DIR"
|
| 258 |
+
echo "Video record build workers: $VIDEO_RECORD_BUILD_WORKERS"
|
| 259 |
+
echo "Video record build chunk size: $VIDEO_RECORD_BUILD_CHUNK_SIZE"
|
| 260 |
+
|
| 261 |
+
python scripts/data/prebuild_multimodal_records_cache.py \
|
| 262 |
+
--tasks="$PREBUILD_TASKS" \
|
| 263 |
+
--splits="$PREBUILD_SPLITS" \
|
| 264 |
+
--action-chunk-size=$ACTION_CHUNK_SIZE \
|
| 265 |
+
--num-past-frames=$NUM_PAST_FRAMES \
|
| 266 |
+
--omni-hidden-root="$PSI_OMNI_OSS_ROOT" \
|
| 267 |
+
--text-root-dir="$PSI_SEED_ROOT" \
|
| 268 |
+
--text-metadata-csv="$PSI_SEED_METADATA" \
|
| 269 |
+
--text-records-cache-dir="$PSI_TEXT_RECORDS_CACHE_DIR" \
|
| 270 |
+
--text-split-source-namespace=seed \
|
| 271 |
+
--audio-sources-json="$PSI_AUDIO_SOURCES_JSON" \
|
| 272 |
+
--audio-records-cache-dir="$PSI_AUDIO_RECORDS_CACHE_DIR" \
|
| 273 |
+
--audio-sample-stride=1 \
|
| 274 |
+
--audio-motion-fps=50 \
|
| 275 |
+
--audio-audio-sampling-rate=16000 \
|
| 276 |
+
--audio-audio-window-seconds=1.0 \
|
| 277 |
+
--audio-audio-left-context-seconds=0.0 \
|
| 278 |
+
--no-audio-full-audio \
|
| 279 |
+
--audio-command-sources-json="$PSI_AUDIO_COMMAND_SOURCES_JSON" \
|
| 280 |
+
--audio-command-records-cache-dir="$PSI_AUDIO_COMMAND_RECORDS_CACHE_DIR" \
|
| 281 |
+
--audio-command-sample-stride=1 \
|
| 282 |
+
--audio-command-motion-fps=50 \
|
| 283 |
+
--audio-command-audio-sampling-rate=16000 \
|
| 284 |
+
--audio-command-audio-window-seconds=1.0 \
|
| 285 |
+
--audio-command-audio-left-context-seconds=0.0 \
|
| 286 |
+
--audio-command-full-audio \
|
| 287 |
+
--video-sources-json="$PSI_VIDEO_SOURCES_JSON" \
|
| 288 |
+
--video-sample-stride=1 \
|
| 289 |
+
--video-motion-fps=50 \
|
| 290 |
+
--video-video-sample-fps=$VIDEO_SAMPLE_FPS \
|
| 291 |
+
--video-video-size-pixels=$VIDEO_SIZE_PIXELS \
|
| 292 |
+
--video-video-decoder=$VIDEO_DECODER \
|
| 293 |
+
--video-record-build-workers=$VIDEO_RECORD_BUILD_WORKERS \
|
| 294 |
+
--video-record-build-chunk-size=$VIDEO_RECORD_BUILD_CHUNK_SIZE \
|
| 295 |
+
--video-clip-sources-json="$PSI_VIDEO_CLIP_SOURCES_JSON" \
|
| 296 |
+
--video-clip-records-cache-dir="$VIDEO_CLIP_RECORDS_CACHE_DIR" \
|
| 297 |
+
--video-clip-sample-stride=1 \
|
| 298 |
+
--video-clip-motion-fps=50 \
|
| 299 |
+
--video-clip-video-sample-fps=$VIDEO_CLIP_SAMPLE_FPS \
|
| 300 |
+
--video-clip-video-size-pixels=$VIDEO_SIZE_PIXELS \
|
| 301 |
+
--video-clip-video-decoder=$VIDEO_DECODER \
|
| 302 |
+
--video-clip-record-build-workers=$VIDEO_RECORD_BUILD_WORKERS \
|
| 303 |
+
--video-clip-record-build-chunk-size=$VIDEO_RECORD_BUILD_CHUNK_SIZE \
|
| 304 |
+
$PREBUILD_VIDEO_CLIP_TRUST_METADATA_FLAG \
|
| 305 |
+
--video-clip-action-exec-horizon=$ACTION_EXEC_HORIZON \
|
| 306 |
+
"$@"
|
prepare_audio_command_hidden.sh
ADDED
|
@@ -0,0 +1,43 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#!/usr/bin/env bash
|
| 2 |
+
set -Eeuo pipefail
|
| 3 |
+
|
| 4 |
+
PROJECT_ROOT=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
|
| 5 |
+
cd "$PROJECT_ROOT"
|
| 6 |
+
|
| 7 |
+
if [[ $# -ne 0 ]]; then
|
| 8 |
+
echo "本脚本不接收位置参数;请使用 HFD_AUDIO_COMMAND_* 环境变量覆盖。" >&2
|
| 9 |
+
exit 2
|
| 10 |
+
fi
|
| 11 |
+
|
| 12 |
+
# 本次正式产物使用独立目录;OVERWRITE 始终关闭,重复执行只允许安全续跑。
|
| 13 |
+
OUTPUT_ROOT=${HFD_AUDIO_COMMAND_OUTPUT_ROOT:-/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/videogen/hfd_audio_command}
|
| 14 |
+
GPU_ID=${HFD_AUDIO_COMMAND_GPU:-0}
|
| 15 |
+
BATCH_SIZE=${HFD_AUDIO_COMMAND_BATCH_SIZE:-512}
|
| 16 |
+
WORKERS_PER_GPU=${HFD_AUDIO_COMMAND_WORKERS_PER_GPU:-1}
|
| 17 |
+
|
| 18 |
+
export PSI_PREP_OUTPUT_KIND=hidden
|
| 19 |
+
export PSI_PREP_OUTPUT_ROOT="$OUTPUT_ROOT"
|
| 20 |
+
export PSI_PREP_TASKS=audio_command_text
|
| 21 |
+
export PSI_PREP_SOURCES=qwen3_tts_seed_audio_command
|
| 22 |
+
export PSI_PREP_CUDA_VISIBLE_DEVICES="$GPU_ID"
|
| 23 |
+
export PSI_PREP_WORKERS_PER_GPU="$WORKERS_PER_GPU"
|
| 24 |
+
export PSI_PREP_HIDDEN_BATCH_SIZE="$BATCH_SIZE"
|
| 25 |
+
# 本入口只负责“音频切片 -> hidden states”;motion source-split 另行构建,
|
| 26 |
+
# 避免为一次 hidden 导出额外扫描 14 万条 motion。
|
| 27 |
+
export PSI_PREP_BUILD_SOURCE_SPLIT=0
|
| 28 |
+
export PSI_PREP_CLEAN_STAGING=1
|
| 29 |
+
export PSI_PREP_OVERWRITE=0
|
| 30 |
+
export PSI_PREP_LIMIT=0
|
| 31 |
+
export PSI_PREP_AUDIO_COMMAND_FULL_AUDIO=0
|
| 32 |
+
export PSI_PREP_AUDIO_COMMAND_SAMPLE_STEP_FRAMES=20
|
| 33 |
+
export PSI_PREP_AUDIO_COMMAND_WINDOW_SECONDS=0.6
|
| 34 |
+
export PSI_PREP_VERIFY_CANONICAL_COUNTS=1
|
| 35 |
+
export PSI_PREP_PYTHON_BIN="$PROJECT_ROOT/.venv-psi/bin/python"
|
| 36 |
+
|
| 37 |
+
echo "[audio-command] input: Qwen3-TTS WAV"
|
| 38 |
+
echo "[audio-command] window: 0.6s / 30 motion frames"
|
| 39 |
+
echo "[audio-command] step: 0.4s / 20 motion frames"
|
| 40 |
+
echo "[audio-command] output: $OUTPUT_ROOT"
|
| 41 |
+
echo "[audio-command] GPU: $GPU_ID"
|
| 42 |
+
|
| 43 |
+
exec bash "$PROJECT_ROOT/prepare_omni_training_data.sh"
|
prepare_full_duplex_continuous_audio.sh
ADDED
|
@@ -0,0 +1,326 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#!/usr/bin/env bash
|
| 2 |
+
set -Eeuo pipefail
|
| 3 |
+
|
| 4 |
+
PROJECT_ROOT=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
|
| 5 |
+
cd "$PROJECT_ROOT"
|
| 6 |
+
source "$PROJECT_ROOT/scripts/activate_psi_env.sh"
|
| 7 |
+
export PYTHONPATH="$PROJECT_ROOT/src${PYTHONPATH:+:$PYTHONPATH}"
|
| 8 |
+
|
| 9 |
+
DATA_ROOT=${HFD_CONTINUOUS_DATA_ROOT:-/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/videogen/hfd_audio_command_continuous_office_v1}
|
| 10 |
+
OLD_HIDDEN_ROOT=${HFD_OLD_AUDIO_COMMAND_ROOT:-/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/videogen/hfd_audio_command_source}
|
| 11 |
+
BACKGROUND_ROOT="$DATA_ROOT/background"
|
| 12 |
+
SESSION_DIR="$BACKGROUND_ROOT/sessions"
|
| 13 |
+
SESSION_MANIFEST="$SESSION_DIR/session_manifest.json"
|
| 14 |
+
MIXED_HIDDEN_ROOT="$DATA_ROOT/mixed_hidden"
|
| 15 |
+
BACKGROUND_HIDDEN_ROOT="$DATA_ROOT/background_hidden"
|
| 16 |
+
MANIFEST_ROOT="$DATA_ROOT/manifests"
|
| 17 |
+
ASSIGNMENT_PATH="$MANIFEST_ROOT/source_background_assignments.jsonl"
|
| 18 |
+
MIXED_SOURCE_CONFIG="$MANIFEST_ROOT/mixed_source_config.json"
|
| 19 |
+
BACKGROUND_SOURCE_CONFIG="$MANIFEST_ROOT/background_source_config.json"
|
| 20 |
+
MODEL_PATH=${HFD_QWEN_BACKBONE:-/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/team_workspace/public_ckpt/Qwen/Qwen2.5-Omni-3B}
|
| 21 |
+
DEMAND_ROOT=${HFD_DEMAND_ROOT:-$PROJECT_ROOT/data/full_duplex_background_review_260730/raw}
|
| 22 |
+
CUDA_VISIBLE_DEVICES=${CUDA_VISIBLE_DEVICES:-0,1,2,3}
|
| 23 |
+
WORKERS_PER_GPU=${HFD_CONTINUOUS_WORKERS_PER_GPU:-4}
|
| 24 |
+
CPU_WORKERS=${HFD_CONTINUOUS_CPU_WORKERS:-32}
|
| 25 |
+
LIMIT_SOURCES=${HFD_CONTINUOUS_LIMIT_SOURCES:-0}
|
| 26 |
+
VERIFY_SPEECH_SOURCES=${HFD_CONTINUOUS_VERIFY_SPEECH_SOURCES:-64}
|
| 27 |
+
VERIFY_BACKGROUND_SOURCES=${HFD_CONTINUOUS_VERIFY_BACKGROUND_SOURCES:-10}
|
| 28 |
+
VERIFY_WINDOWS_PER_BACKGROUND=${HFD_CONTINUOUS_VERIFY_BACKGROUND_WINDOWS:-8}
|
| 29 |
+
FORCE_WAVEFORM_BUILD=${HFD_CONTINUOUS_FORCE_WAVEFORM_BUILD:-0}
|
| 30 |
+
OVERWRITE_HIDDEN=${HFD_CONTINUOUS_OVERWRITE_HIDDEN:-0}
|
| 31 |
+
TORCHRUN_LOCAL_ADDR=${HFD_TORCHRUN_LOCAL_ADDR:-127.0.0.1}
|
| 32 |
+
MONITOR_STOP=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/monitor/stop.sh
|
| 33 |
+
MONITOR_START=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/monitor/cmd.sh
|
| 34 |
+
LOG_DIR="$DATA_ROOT/logs"
|
| 35 |
+
LOG_PATH="$LOG_DIR/prepare.log"
|
| 36 |
+
|
| 37 |
+
mkdir -p "$LOG_DIR"
|
| 38 |
+
exec > >(tee -a "$LOG_PATH") 2>&1
|
| 39 |
+
|
| 40 |
+
echo "================================================================================"
|
| 41 |
+
echo "HFD 连续办公室背景数据"
|
| 42 |
+
echo "结果根目录: $DATA_ROOT"
|
| 43 |
+
echo "混合语音 hidden: $MIXED_HIDDEN_ROOT"
|
| 44 |
+
echo "连续背景 hidden: $BACKGROUND_HIDDEN_ROOT"
|
| 45 |
+
echo "GPU: $CUDA_VISIBLE_DEVICES"
|
| 46 |
+
echo "每卡 B=1 进程: $WORKERS_PER_GPU"
|
| 47 |
+
echo "source 限制: $LIMIT_SOURCES"
|
| 48 |
+
echo "================================================================================"
|
| 49 |
+
|
| 50 |
+
if [[ "$LIMIT_SOURCES" != "0" \
|
| 51 |
+
&& "$DATA_ROOT" == "/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/videogen/hfd_audio_command_continuous_office_v1" ]]; then
|
| 52 |
+
echo "[error] 有限 source 测试不能写进正式目录。" >&2
|
| 53 |
+
exit 2
|
| 54 |
+
fi
|
| 55 |
+
for required in \
|
| 56 |
+
"$MODEL_PATH/config.json" \
|
| 57 |
+
"$DEMAND_ROOT/OOFFICE_ch01.wav" \
|
| 58 |
+
"$DEMAND_ROOT/OHALLWAY_ch01.wav"; do
|
| 59 |
+
if [[ ! -e "$required" ]]; then
|
| 60 |
+
echo "[error] 缺少输入: $required" >&2
|
| 61 |
+
exit 2
|
| 62 |
+
fi
|
| 63 |
+
done
|
| 64 |
+
|
| 65 |
+
if [[ "$FORCE_WAVEFORM_BUILD" == "1" \
|
| 66 |
+
|| ! -f "$DATA_ROOT/reports/waveform_build.json" ]]; then
|
| 67 |
+
if [[ ! -d "$OLD_HIDDEN_ROOT" && ! -f "$ASSIGNMENT_PATH" ]]; then
|
| 68 |
+
echo "[error] 首次构造需要旧 cache 的有效 source 清单;重建则可复用新 assignment。" >&2
|
| 69 |
+
echo " legacy=$OLD_HIDDEN_ROOT" >&2
|
| 70 |
+
echo " fallback=$ASSIGNMENT_PATH" >&2
|
| 71 |
+
exit 2
|
| 72 |
+
fi
|
| 73 |
+
fi
|
| 74 |
+
|
| 75 |
+
if [[ ! -f "$SESSION_MANIFEST" ]]; then
|
| 76 |
+
python -u scripts/data/generate_office_background_candidates.py \
|
| 77 |
+
--production-sessions \
|
| 78 |
+
--output-dir "$SESSION_DIR" \
|
| 79 |
+
--duration-seconds 600 \
|
| 80 |
+
--seed 260730 \
|
| 81 |
+
--demand-root "$DEMAND_ROOT" \
|
| 82 |
+
--vad-margin-seconds 1.0 \
|
| 83 |
+
--minimum-real-session-seconds 20.0
|
| 84 |
+
fi
|
| 85 |
+
|
| 86 |
+
WAVEFORM_ARGS=(
|
| 87 |
+
scripts/data/build_continuous_audio_command_sources.py
|
| 88 |
+
--output-root "$DATA_ROOT"
|
| 89 |
+
--background-manifest "$SESSION_MANIFEST"
|
| 90 |
+
--old-hidden-root "$OLD_HIDDEN_ROOT"
|
| 91 |
+
--workers "$CPU_WORKERS"
|
| 92 |
+
--limit-sources "$LIMIT_SOURCES"
|
| 93 |
+
--initial-idle-max-steps 5
|
| 94 |
+
--stationary-blend-frames 20
|
| 95 |
+
--safety-steps 10
|
| 96 |
+
--seed 260730
|
| 97 |
+
)
|
| 98 |
+
if [[ "$FORCE_WAVEFORM_BUILD" == "1" \
|
| 99 |
+
|| ! -f "$DATA_ROOT/reports/waveform_build.json" ]]; then
|
| 100 |
+
python -u "${WAVEFORM_ARGS[@]}" --mode build
|
| 101 |
+
fi
|
| 102 |
+
python -u "${WAVEFORM_ARGS[@]}" --mode verify
|
| 103 |
+
|
| 104 |
+
monitor_stopped=0
|
| 105 |
+
restore_monitor() {
|
| 106 |
+
if [[ "$monitor_stopped" == "1" && -f "$MONITOR_START" ]]; then
|
| 107 |
+
bash "$MONITOR_START" || true
|
| 108 |
+
monitor_stopped=0
|
| 109 |
+
fi
|
| 110 |
+
}
|
| 111 |
+
trap restore_monitor EXIT
|
| 112 |
+
if [[ -f "$MONITOR_STOP" ]]; then
|
| 113 |
+
bash "$MONITOR_STOP"
|
| 114 |
+
monitor_stopped=1
|
| 115 |
+
fi
|
| 116 |
+
|
| 117 |
+
export CUDA_VISIBLE_DEVICES
|
| 118 |
+
export NCCL_DEBUG=WARN
|
| 119 |
+
export PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True
|
| 120 |
+
VISIBLE_GPU_COUNT=$(python - <<'PY'
|
| 121 |
+
import torch
|
| 122 |
+
print(torch.cuda.device_count())
|
| 123 |
+
PY
|
| 124 |
+
)
|
| 125 |
+
if [[ "$VISIBLE_GPU_COUNT" != "4" ]]; then
|
| 126 |
+
echo "[error] 本次正式生成要求恰好四张可见 GPU,实际为 $VISIBLE_GPU_COUNT。" >&2
|
| 127 |
+
exit 2
|
| 128 |
+
fi
|
| 129 |
+
if [[ ! "$WORKERS_PER_GPU" =~ ^[1-9][0-9]*$ ]]; then
|
| 130 |
+
echo "[error] WORKERS_PER_GPU 必须是正整数。" >&2
|
| 131 |
+
exit 2
|
| 132 |
+
fi
|
| 133 |
+
NPROC=$((VISIBLE_GPU_COUNT * WORKERS_PER_GPU))
|
| 134 |
+
|
| 135 |
+
export_hidden() {
|
| 136 |
+
local output_root=$1
|
| 137 |
+
local source_name=$2
|
| 138 |
+
local source_config=$3
|
| 139 |
+
local limit=$4
|
| 140 |
+
local overwrite_args=()
|
| 141 |
+
local limit_args=()
|
| 142 |
+
if [[ "$OVERWRITE_HIDDEN" == "1" ]]; then
|
| 143 |
+
overwrite_args+=(--overwrite)
|
| 144 |
+
fi
|
| 145 |
+
if [[ "$limit" != "0" ]]; then
|
| 146 |
+
limit_args+=(--limit-sources "$limit")
|
| 147 |
+
fi
|
| 148 |
+
python -m torch.distributed.run \
|
| 149 |
+
--standalone \
|
| 150 |
+
--local_addr="$TORCHRUN_LOCAL_ADDR" \
|
| 151 |
+
--nproc_per_node="$NPROC" \
|
| 152 |
+
scripts/data/cache_omni_hidden_states.py \
|
| 153 |
+
--mode export \
|
| 154 |
+
--model-path "$MODEL_PATH" \
|
| 155 |
+
--output-root "$output_root" \
|
| 156 |
+
--tasks audio_command_text \
|
| 157 |
+
--sources "$source_name" \
|
| 158 |
+
--source-config-file "$source_config" \
|
| 159 |
+
--motion-fps 50 \
|
| 160 |
+
--audio-command-sample-step-frames 20 \
|
| 161 |
+
--audio-command-window-seconds 0.6 \
|
| 162 |
+
--audio-sampling-rate 16000 \
|
| 163 |
+
--audio-batch-size 1 \
|
| 164 |
+
--num-io-workers 2 \
|
| 165 |
+
--num-save-workers 4 \
|
| 166 |
+
--max-pending-save-batches 8 \
|
| 167 |
+
--save-dtype bf16 \
|
| 168 |
+
--storage-format source \
|
| 169 |
+
--no-audio-command-full-audio \
|
| 170 |
+
"${limit_args[@]}" \
|
| 171 |
+
"${overwrite_args[@]}"
|
| 172 |
+
}
|
| 173 |
+
|
| 174 |
+
verify_hidden() {
|
| 175 |
+
local output_root=$1
|
| 176 |
+
local source_name=$2
|
| 177 |
+
local source_config=$3
|
| 178 |
+
local verify_sources=$4
|
| 179 |
+
local verify_windows=$5
|
| 180 |
+
CUDA_VISIBLE_DEVICES=0 python -u scripts/data/cache_omni_hidden_states.py \
|
| 181 |
+
--mode verify \
|
| 182 |
+
--model-path "$MODEL_PATH" \
|
| 183 |
+
--output-root "$output_root" \
|
| 184 |
+
--tasks audio_command_text \
|
| 185 |
+
--sources "$source_name" \
|
| 186 |
+
--source-config-file "$source_config" \
|
| 187 |
+
--motion-fps 50 \
|
| 188 |
+
--audio-command-sample-step-frames 20 \
|
| 189 |
+
--audio-command-window-seconds 0.6 \
|
| 190 |
+
--audio-sampling-rate 16000 \
|
| 191 |
+
--audio-batch-size 1 \
|
| 192 |
+
--save-dtype bf16 \
|
| 193 |
+
--storage-format source \
|
| 194 |
+
--verify-sources "$verify_sources" \
|
| 195 |
+
--verify-windows-per-source "$verify_windows" \
|
| 196 |
+
--no-audio-command-full-audio
|
| 197 |
+
}
|
| 198 |
+
|
| 199 |
+
echo "[GPU] 导出连续纯背景 hidden"
|
| 200 |
+
export_hidden \
|
| 201 |
+
"$BACKGROUND_HIDDEN_ROOT" \
|
| 202 |
+
hfd_continuous_office_background \
|
| 203 |
+
"$BACKGROUND_SOURCE_CONFIG" \
|
| 204 |
+
0
|
| 205 |
+
verify_hidden \
|
| 206 |
+
"$BACKGROUND_HIDDEN_ROOT" \
|
| 207 |
+
hfd_continuous_office_background \
|
| 208 |
+
"$BACKGROUND_SOURCE_CONFIG" \
|
| 209 |
+
"$VERIFY_BACKGROUND_SOURCES" \
|
| 210 |
+
"$VERIFY_WINDOWS_PER_BACKGROUND"
|
| 211 |
+
|
| 212 |
+
echo "[GPU] 导出人声+连续背景 mixed hidden"
|
| 213 |
+
export_hidden \
|
| 214 |
+
"$MIXED_HIDDEN_ROOT" \
|
| 215 |
+
qwen3_tts_seed_audio_command \
|
| 216 |
+
"$MIXED_SOURCE_CONFIG" \
|
| 217 |
+
"$LIMIT_SOURCES"
|
| 218 |
+
verify_hidden \
|
| 219 |
+
"$MIXED_HIDDEN_ROOT" \
|
| 220 |
+
qwen3_tts_seed_audio_command \
|
| 221 |
+
"$MIXED_SOURCE_CONFIG" \
|
| 222 |
+
"$VERIFY_SPEECH_SOURCES" \
|
| 223 |
+
0
|
| 224 |
+
|
| 225 |
+
python - "$DATA_ROOT" "$MODEL_PATH" <<'PY'
|
| 226 |
+
import hashlib
|
| 227 |
+
import json
|
| 228 |
+
import sqlite3
|
| 229 |
+
import sys
|
| 230 |
+
import time
|
| 231 |
+
from pathlib import Path
|
| 232 |
+
|
| 233 |
+
from psi.data.seed.full_duplex_audio_contract import (
|
| 234 |
+
validate_encoder_cache,
|
| 235 |
+
write_json_atomic,
|
| 236 |
+
)
|
| 237 |
+
|
| 238 |
+
root = Path(sys.argv[1]).resolve()
|
| 239 |
+
model = Path(sys.argv[2]).resolve()
|
| 240 |
+
waveform = json.loads((root / "reports" / "waveform_build.json").read_text())
|
| 241 |
+
|
| 242 |
+
def index_totals(cache_root, source_name):
|
| 243 |
+
source_dir = (
|
| 244 |
+
cache_root / "audio" / source_name / "audio_command_text" / "sources"
|
| 245 |
+
)
|
| 246 |
+
rows = {}
|
| 247 |
+
for path in source_dir.glob("index.ws*.rank*.sqlite"):
|
| 248 |
+
connection = sqlite3.connect(f"file:{path}?mode=ro", uri=True)
|
| 249 |
+
try:
|
| 250 |
+
values = connection.execute(
|
| 251 |
+
"SELECT key, path, count FROM sources"
|
| 252 |
+
).fetchall()
|
| 253 |
+
finally:
|
| 254 |
+
connection.close()
|
| 255 |
+
for key, relative_path, count in values:
|
| 256 |
+
value = (str(relative_path), int(count))
|
| 257 |
+
previous = rows.get(str(key))
|
| 258 |
+
if previous is not None and previous != value:
|
| 259 |
+
raise RuntimeError(
|
| 260 |
+
f"Conflicting duplicate source index row: {key}: "
|
| 261 |
+
f"{previous} != {value}"
|
| 262 |
+
)
|
| 263 |
+
rows[str(key)] = value
|
| 264 |
+
return len(rows), sum(count for _, count in rows.values())
|
| 265 |
+
|
| 266 |
+
mixed_root = root / "mixed_hidden"
|
| 267 |
+
background_root = root / "background_hidden"
|
| 268 |
+
mixed_contract = validate_encoder_cache(mixed_root, model)
|
| 269 |
+
background_contract = validate_encoder_cache(background_root, model)
|
| 270 |
+
mixed_sources, mixed_windows = index_totals(
|
| 271 |
+
mixed_root, "qwen3_tts_seed_audio_command"
|
| 272 |
+
)
|
| 273 |
+
background_sources, background_windows = index_totals(
|
| 274 |
+
background_root, "hfd_continuous_office_background"
|
| 275 |
+
)
|
| 276 |
+
expected_sources = int(waveform["source_count"])
|
| 277 |
+
expected_mixed = int(waveform["mixed_window_count"])
|
| 278 |
+
expected_background_sources = int(waveform["background_session_count"])
|
| 279 |
+
expected_background = int(waveform["background_window_count"])
|
| 280 |
+
actual = (
|
| 281 |
+
mixed_sources,
|
| 282 |
+
mixed_windows,
|
| 283 |
+
background_sources,
|
| 284 |
+
background_windows,
|
| 285 |
+
)
|
| 286 |
+
expected = (
|
| 287 |
+
expected_sources,
|
| 288 |
+
expected_mixed,
|
| 289 |
+
expected_background_sources,
|
| 290 |
+
expected_background,
|
| 291 |
+
)
|
| 292 |
+
if actual != expected:
|
| 293 |
+
raise RuntimeError(f"Final cache totals mismatch: actual={actual} expected={expected}")
|
| 294 |
+
|
| 295 |
+
assignment = Path(waveform["assignment_path"])
|
| 296 |
+
digest = hashlib.sha256(assignment.read_bytes()).hexdigest()
|
| 297 |
+
if digest != waveform["assignment_sha256"]:
|
| 298 |
+
raise RuntimeError("Assignment manifest changed before finalization")
|
| 299 |
+
write_json_atomic(
|
| 300 |
+
root / "reports" / "production_complete.json",
|
| 301 |
+
{
|
| 302 |
+
"format": "hfd_continuous_office_production_v1",
|
| 303 |
+
"completed_at": time.time(),
|
| 304 |
+
"data_root": str(root),
|
| 305 |
+
"source_count": mixed_sources,
|
| 306 |
+
"mixed_window_count": mixed_windows,
|
| 307 |
+
"background_source_count": background_sources,
|
| 308 |
+
"background_window_count": background_windows,
|
| 309 |
+
"initial_idle_max_steps": int(waveform["initial_idle_max_steps"]),
|
| 310 |
+
"stationary_blend_frames": int(waveform["stationary_blend_frames"]),
|
| 311 |
+
"safety_steps": int(waveform["safety_steps"]),
|
| 312 |
+
"assignment_seed": int(waveform["seed"]),
|
| 313 |
+
"assignment_path": str(assignment),
|
| 314 |
+
"assignment_sha256": digest,
|
| 315 |
+
"online_encoder_contract": mixed_contract,
|
| 316 |
+
"background_encoder_contract": background_contract,
|
| 317 |
+
},
|
| 318 |
+
)
|
| 319 |
+
print(
|
| 320 |
+
f"[complete] sources={mixed_sources:,} mixed={mixed_windows:,} "
|
| 321 |
+
f"background={background_windows:,} root={root}"
|
| 322 |
+
)
|
| 323 |
+
PY
|
| 324 |
+
|
| 325 |
+
restore_monitor
|
| 326 |
+
echo "[done] $DATA_ROOT"
|
pyproject.toml
ADDED
|
@@ -0,0 +1,85 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
[project]
|
| 2 |
+
name = "psi"
|
| 3 |
+
version = "0.0.0"
|
| 4 |
+
description = "Psi: towards humanoid foundation model"
|
| 5 |
+
readme = "README.md"
|
| 6 |
+
requires-python = "==3.10.*"
|
| 7 |
+
dependencies = [
|
| 8 |
+
"albumentations==1.4.18",
|
| 9 |
+
"datasets==3.6.0",
|
| 10 |
+
"dm-tree==0.1.8",
|
| 11 |
+
"einops==0.8.1",
|
| 12 |
+
"einx>=0.3.0",
|
| 13 |
+
"h5py==3.14.0",
|
| 14 |
+
"imageio==2.34.2",
|
| 15 |
+
"jsonlines>=4.0.0",
|
| 16 |
+
"numpy<2.0.0",
|
| 17 |
+
"numpydantic==1.6.7",
|
| 18 |
+
"omegaconf>=2.3.0",
|
| 19 |
+
"opencv-python",
|
| 20 |
+
"plotly==6.2.0",
|
| 21 |
+
"pydantic==2.10.6",
|
| 22 |
+
"pydantic-yaml>=1.6.0",
|
| 23 |
+
"tqdm==4.67.1",
|
| 24 |
+
"typer>=0.19.1",
|
| 25 |
+
"tyro==0.9.32",
|
| 26 |
+
"vuer[all]>=0.0.68",
|
| 27 |
+
"wandb>=0.20.0",
|
| 28 |
+
"simplejpeg>=1.9.0",
|
| 29 |
+
"dotenv",
|
| 30 |
+
"gdown>=5.2.0",
|
| 31 |
+
]
|
| 32 |
+
|
| 33 |
+
[build-system]
|
| 34 |
+
requires = ["setuptools>=61.0"]
|
| 35 |
+
build-backend = "setuptools.build_meta"
|
| 36 |
+
|
| 37 |
+
[tool.setuptools.package-data]
|
| 38 |
+
psi = ["assets/*"]
|
| 39 |
+
|
| 40 |
+
[tool.uv.sources]
|
| 41 |
+
psi = { path = ".", editable = true }
|
| 42 |
+
simple = { path = "third_party/SIMPLE", editable = true }
|
| 43 |
+
|
| 44 |
+
[tool.uv.workspace]
|
| 45 |
+
members = ["packages/*"]
|
| 46 |
+
|
| 47 |
+
[dependency-groups]
|
| 48 |
+
serve = [
|
| 49 |
+
"fastapi",
|
| 50 |
+
"uvicorn",
|
| 51 |
+
]
|
| 52 |
+
|
| 53 |
+
viz = [
|
| 54 |
+
"transforms3d>=0.4.2",
|
| 55 |
+
"vuer[all]>=0.0.68",
|
| 56 |
+
"viser>=1.0.15",
|
| 57 |
+
#"pin>=3.8.0",
|
| 58 |
+
#"numpy<2.0.0",
|
| 59 |
+
]
|
| 60 |
+
|
| 61 |
+
psi = [
|
| 62 |
+
"setuptools>=61",
|
| 63 |
+
"torch==2.7.0",
|
| 64 |
+
"torchvision==0.22.0",
|
| 65 |
+
"torchcodec==0.4.0",
|
| 66 |
+
"transformers==4.57.0",
|
| 67 |
+
"deepspeed==0.17.1",
|
| 68 |
+
"flash_attn==2.7.4.post1",
|
| 69 |
+
"triton==3.3.0",
|
| 70 |
+
"accelerate==1.7.0",
|
| 71 |
+
"peft==0.17.1",
|
| 72 |
+
"qwen-vl-utils==0.0.14",
|
| 73 |
+
"sentencepiece",
|
| 74 |
+
"lerobot @ git+https://github.com/songlin/lerobot.git@09929d8057b044b53aecaf5c6d7eb71f99e8beb9"
|
| 75 |
+
]
|
| 76 |
+
|
| 77 |
+
#simple = [
|
| 78 |
+
# "simple[full]",
|
| 79 |
+
#]
|
| 80 |
+
|
| 81 |
+
[tool.uv.extra-build-dependencies]
|
| 82 |
+
flash-attn = [{ requirement = "torch", match-runtime = true }]
|
| 83 |
+
|
| 84 |
+
[project.scripts]
|
| 85 |
+
serve_psi0 = "psi.deploy.psi0_serve_simple:main"
|
uv.lock
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|