Download backup/prepare_omni_training_data.sh from Tinghong-Ye22/code-HFD: direct link, hf CLI and curl.
- Browser
- Download file 50.8 kB
-
https://huggingface.co/Tinghong-Ye22/code-HFD/resolve/main/backup/prepare_omni_training_data.sh
- Command line
-
hf download hf://Tinghong-Ye22/code-HFD/backup/prepare_omni_training_data.sh
-
curl -L -o prepare_omni_training_data.sh https://huggingface.co/Tinghong-Ye22/code-HFD/resolve/main/backup/prepare_omni_training_data.sh
50.8 kB
#!/usr/bin/env bash
set -Eeuo pipefail
# Archived data builder; keep relative dependencies rooted at the HFD project.
PROJECT_ROOT=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)
cd "$PROJECT_ROOT"
# =============================================================================
# 只需要改这一段
# =============================================================================
# 默认只生成 HFD audio-command 需要的 hidden:一个完整 WAV/source
# 对应一个紧凑 safetensors,内部仍是 0.6 秒窗口、0.4 秒滑动。
# 不生成逐窗口文件,不做大分片 staging/merge,也不构造无关任务。
# 旧的多任务 OSS 大分片流程可显式设置 HIDDEN_LAYOUT=oss 后继续复用。
OUTPUT_KIND="hidden" # embeds | hidden
OUTPUT_ROOT="/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/videogen/hfd_audio_command_source"
HIDDEN_LAYOUT="source" # source | oss
# 统一音频入口是 WAV。Qwen3-TTS 直接读取原始 seed_audio/*.wav;dance/2264
# 原始数据只有 MP4 音轨,因此先把历史 manifest 的规范采样(NPY/少量 MP4 fallback)
# 无损封装成 float32 WAV。
# 这样既不改变旧 hidden 的数值,又能让后续音频切 clip 只面对一种输入格式。
AUDIO_WAV_ROOT="/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/videogen/psi0_audio_wav_v26"
# 可选任务:text,audio_text,audio_only,audio_command_text,audio_command_only,
# video_clip_text,video_clip_only
SELECTED_TASKS="audio_command_text"
# 可选数据源:
# seed_text
# psi0_dance_audio
# 2264_with_audio
# qwen3_tts_seed_audio_command
# dance_1723_video_10s_clip50
# motion_example_0506_demo_clip50
# 2264_with_audio_clip50
# distilled_video_0226_clip50(目前仅建议用于 embeds;正式训练配置尚未引用它)
SELECTED_SOURCES="qwen3_tts_seed_audio_command"
CUDA_VISIBLE_DEVICES="${CUDA_VISIBLE_DEVICES:-0,1,2,3,4,5,6,7}" # embeds 模式不用 GPU
WORKERS_PER_GPU=8
TORCHRUN_LOCAL_ADDR="127.0.0.1"
BUILD_SOURCE_SPLIT=0 # HFD source hidden 不需要旧多任务 split 包
CLEAN_STAGING_AFTER_SUCCESS=1 # hidden 成功后删除中间大 shard 和 video embeds
OVERWRITE=0
# 下面通常不用改。正式数据必须保持 LIMIT=0;非零只用于逐阶段调试,
# 其输出不应直接交给训练。
LIMIT=0
VIDEO_METADATA_SHARDS=4
VIDEO_BUILD_WORKERS=64
VIDEO_EMBED_WORKERS=32
VIDEO_SAMPLE_FPS=8
VIDEO_SIZE_PIXELS=50176
VIDEO_DECODER="ffmpeg"
AUDIO_WAV_WORKERS=32
AUDIO_SAMPLE_STEP_FRAMES=1
AUDIO_WINDOW_SECONDS=0.6
# audio_command 默认不再输入整段 WAV:只保留完整的 0.6 秒窗口,
# 每 20 个 50 FPS 动作帧(0.4 秒)向后滑动一次。
AUDIO_COMMAND_FULL_AUDIO=0 # 0:切片 | 1:旧版整段语音
AUDIO_COMMAND_SAMPLE_STEP_FRAMES=20
AUDIO_COMMAND_WINDOW_SECONDS=0.6
HIDDEN_BATCH_SIZE=1
HIDDEN_SHARD_SIZE=4096
# metadata:严格复现旧 cache_omni_hidden_states_v23.sh 的样本范围。
# motion_csv:按动作 CSV 的真实帧数生成,属于新的数据定义。
AUDIO_FRAME_COUNT_SOURCE="metadata" # metadata | motion_csv
FINAL_SAMPLES_PER_SHARD=4096
FINAL_MAX_SHARD_GIB=4.5
SHUFFLE_SEED=20260710
MERGE_WORKERS=32
MERGE_SOURCE_READER_CACHE=32
VERIFY_SAMPLES=64
ONLINE_VERIFY_SOURCES=64 # 前若干完整 WAV 逐窗口复算,要求与在线 B=1 逐元素相等
VERIFY_CANONICAL_COUNTS=1 # 默认全量四任务时核对当前数据定义的精确条数
MANAGE_LOCAL_GPU_MONITOR=1
GPU_MONITOR_SCRIPT="/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/monitor/cmd.sh"
# 自动化/小测试可以用 PSI_PREP_* 临时覆盖;正常使用无需传任何参数。
OUTPUT_KIND=${PSI_PREP_OUTPUT_KIND:-$OUTPUT_KIND}
OUTPUT_ROOT=${PSI_PREP_OUTPUT_ROOT:-$OUTPUT_ROOT}
HIDDEN_LAYOUT=${PSI_PREP_HIDDEN_LAYOUT:-$HIDDEN_LAYOUT}
AUDIO_WAV_ROOT=${PSI_PREP_AUDIO_WAV_ROOT:-$AUDIO_WAV_ROOT}
SELECTED_TASKS=${PSI_PREP_TASKS:-$SELECTED_TASKS}
SELECTED_SOURCES=${PSI_PREP_SOURCES:-$SELECTED_SOURCES}
CUDA_VISIBLE_DEVICES=${PSI_PREP_CUDA_VISIBLE_DEVICES:-$CUDA_VISIBLE_DEVICES}
WORKERS_PER_GPU=${PSI_PREP_WORKERS_PER_GPU:-$WORKERS_PER_GPU}
TORCHRUN_LOCAL_ADDR=${PSI_PREP_TORCHRUN_LOCAL_ADDR:-$TORCHRUN_LOCAL_ADDR}
BUILD_SOURCE_SPLIT=${PSI_PREP_BUILD_SOURCE_SPLIT:-$BUILD_SOURCE_SPLIT}
CLEAN_STAGING_AFTER_SUCCESS=${PSI_PREP_CLEAN_STAGING:-$CLEAN_STAGING_AFTER_SUCCESS}
OVERWRITE=${PSI_PREP_OVERWRITE:-$OVERWRITE}
LIMIT=${PSI_PREP_LIMIT:-$LIMIT}
VIDEO_METADATA_SHARDS=${PSI_PREP_VIDEO_METADATA_SHARDS:-$VIDEO_METADATA_SHARDS}
VIDEO_BUILD_WORKERS=${PSI_PREP_VIDEO_BUILD_WORKERS:-$VIDEO_BUILD_WORKERS}
VIDEO_EMBED_WORKERS=${PSI_PREP_VIDEO_EMBED_WORKERS:-$VIDEO_EMBED_WORKERS}
AUDIO_WAV_WORKERS=${PSI_PREP_AUDIO_WAV_WORKERS:-$AUDIO_WAV_WORKERS}
AUDIO_SAMPLE_STEP_FRAMES=${PSI_PREP_AUDIO_SAMPLE_STEP_FRAMES:-$AUDIO_SAMPLE_STEP_FRAMES}
AUDIO_WINDOW_SECONDS=${PSI_PREP_AUDIO_WINDOW_SECONDS:-$AUDIO_WINDOW_SECONDS}
AUDIO_COMMAND_FULL_AUDIO=${PSI_PREP_AUDIO_COMMAND_FULL_AUDIO:-$AUDIO_COMMAND_FULL_AUDIO}
AUDIO_COMMAND_SAMPLE_STEP_FRAMES=${PSI_PREP_AUDIO_COMMAND_SAMPLE_STEP_FRAMES:-$AUDIO_COMMAND_SAMPLE_STEP_FRAMES}
AUDIO_COMMAND_WINDOW_SECONDS=${PSI_PREP_AUDIO_COMMAND_WINDOW_SECONDS:-$AUDIO_COMMAND_WINDOW_SECONDS}
HIDDEN_BATCH_SIZE=${PSI_PREP_HIDDEN_BATCH_SIZE:-$HIDDEN_BATCH_SIZE}
HIDDEN_SHARD_SIZE=${PSI_PREP_HIDDEN_SHARD_SIZE:-$HIDDEN_SHARD_SIZE}
AUDIO_FRAME_COUNT_SOURCE=${PSI_PREP_AUDIO_FRAME_COUNT_SOURCE:-$AUDIO_FRAME_COUNT_SOURCE}
FINAL_SAMPLES_PER_SHARD=${PSI_PREP_FINAL_SAMPLES_PER_SHARD:-$FINAL_SAMPLES_PER_SHARD}
FINAL_MAX_SHARD_GIB=${PSI_PREP_FINAL_MAX_SHARD_GIB:-$FINAL_MAX_SHARD_GIB}
MERGE_WORKERS=${PSI_PREP_MERGE_WORKERS:-$MERGE_WORKERS}
VERIFY_SAMPLES=${PSI_PREP_VERIFY_SAMPLES:-$VERIFY_SAMPLES}
ONLINE_VERIFY_SOURCES=${PSI_PREP_ONLINE_VERIFY_SOURCES:-$ONLINE_VERIFY_SOURCES}
VERIFY_CANONICAL_COUNTS=${PSI_PREP_VERIFY_CANONICAL_COUNTS:-$VERIFY_CANONICAL_COUNTS}
MANAGE_LOCAL_GPU_MONITOR=${PSI_PREP_MANAGE_GPU_MONITOR:-$MANAGE_LOCAL_GPU_MONITOR}
DRY_RUN=${PSI_PREP_DRY_RUN:-0}
# =============================================================================
# 固定的数据位置和模型配置
# =============================================================================
PYTHON_BIN=${PSI_PREP_PYTHON_BIN:-$PROJECT_ROOT/.venv-psi/bin/python}
MODEL_PATH=${PSI_PREP_MODEL_PATH:-/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/team_workspace/public_ckpt/Qwen/Qwen2.5-Omni-3B}
SEED_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/Psi0/dataset/seed_all
SEED_METADATA=$SEED_ROOT/metadata/seed_metadata_v004.csv
DANCE_MOTION_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/GR00T-WholeBodyControl/gear_sonic_deploy/reference/converted/psi0_dance/gvhmr
DANCE_AV_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/dance_1723_video_psi0/dance_1723_video_10s
MOTION_EXAMPLE_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/videogen/motion_example_0506_demo
AUDIO_2264_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/videogen/2264_with_audio
QWEN3_TTS_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/qwen3-tts
DISTILLED_VIDEO_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/videogen/distilled_video_0226
if [[ $# -ne 0 ]]; then
echo "本脚本不接收位置参数;请直接修改顶部配置。" >&2
exit 2
fi
normalize_csv() {
local value=${1//[[:space:]]/}
value=${value#,}
value=${value%,}
printf '%s' "$value"
}
csv_has() {
local values=",$(normalize_csv "$1"),"
[[ "$values" == *",$2,"* ]]
}
validate_csv_members() {
local label=$1
local values=$2
shift 2
local allowed=",$*,"
allowed=${allowed// /,}
local token
IFS=',' read -r -a tokens <<< "$(normalize_csv "$values")"
for token in "${tokens[@]}"; do
[[ -n "$token" ]] || continue
if [[ "$allowed" != *",$token,"* ]]; then
echo "未知${label}: $token" >&2
exit 2
fi
done
}
SELECTED_TASKS=$(normalize_csv "$SELECTED_TASKS")
SELECTED_SOURCES=$(normalize_csv "$SELECTED_SOURCES")
validate_csv_members "任务" "$SELECTED_TASKS" \
text audio_text audio_only audio_command_text audio_command_only video_clip_text video_clip_only
validate_csv_members "数据源" "$SELECTED_SOURCES" \
seed_text psi0_dance_audio 2264_with_audio qwen3_tts_seed_audio_command \
dance_1723_video_10s_clip50 motion_example_0506_demo_clip50 \
2264_with_audio_clip50 distilled_video_0226_clip50
if [[ "$OUTPUT_KIND" != "embeds" && "$OUTPUT_KIND" != "hidden" ]]; then
echo "OUTPUT_KIND 只能是 embeds 或 hidden" >&2
exit 2
fi
if [[ "$HIDDEN_LAYOUT" != "source" && "$HIDDEN_LAYOUT" != "oss" ]]; then
echo "HIDDEN_LAYOUT 只能是 source 或 oss" >&2
exit 2
fi
if [[ "$AUDIO_FRAME_COUNT_SOURCE" != "metadata" && "$AUDIO_FRAME_COUNT_SOURCE" != "motion_csv" ]]; then
echo "AUDIO_FRAME_COUNT_SOURCE 只能是 metadata 或 motion_csv" >&2
exit 2
fi
if (( AUDIO_WAV_WORKERS <= 0 || AUDIO_SAMPLE_STEP_FRAMES <= 0 || AUDIO_COMMAND_SAMPLE_STEP_FRAMES <= 0 )); then
echo "AUDIO_WAV_WORKERS、AUDIO_SAMPLE_STEP_FRAMES 和 AUDIO_COMMAND_SAMPLE_STEP_FRAMES 必须为正整数" >&2
exit 2
fi
if (( WORKERS_PER_GPU <= 0 || ONLINE_VERIFY_SOURCES <= 0 )); then
echo "WORKERS_PER_GPU 和 ONLINE_VERIFY_SOURCES 必须为正整数" >&2
exit 2
fi
if [[ "$AUDIO_COMMAND_FULL_AUDIO" != "0" && "$AUDIO_COMMAND_FULL_AUDIO" != "1" ]]; then
echo "AUDIO_COMMAND_FULL_AUDIO 只能是 0 或 1" >&2
exit 2
fi
if [[ -z "$SELECTED_TASKS" || -z "$SELECTED_SOURCES" ]]; then
echo "SELECTED_TASKS 和 SELECTED_SOURCES 不能为空" >&2
exit 2
fi
if [[ ! -x "$PYTHON_BIN" ]]; then
echo "Python 不可执行: $PYTHON_BIN" >&2
exit 2
fi
if ! "$PYTHON_BIN" - "$AUDIO_WINDOW_SECONDS" "$AUDIO_COMMAND_WINDOW_SECONDS" <<'PY'
import sys
raise SystemExit(0 if float(sys.argv[1]) > 0 and float(sys.argv[2]) > 0 else 1)
PY
then
echo "AUDIO_WINDOW_SECONDS 和 AUDIO_COMMAND_WINDOW_SECONDS 必须为正数" >&2
exit 2
fi
if csv_has "$SELECTED_TASKS" text && ! csv_has "$SELECTED_SOURCES" seed_text; then
echo "选择了 text 任务,但没有选择 seed_text 数据源" >&2
exit 2
fi
if { csv_has "$SELECTED_TASKS" audio_text || csv_has "$SELECTED_TASKS" audio_only; } \
&& ! csv_has "$SELECTED_SOURCES" psi0_dance_audio \
&& ! csv_has "$SELECTED_SOURCES" 2264_with_audio; then
echo "选择了 audio 任务,但没有选择 audio 数据源" >&2
exit 2
fi
if { csv_has "$SELECTED_TASKS" audio_command_text || csv_has "$SELECTED_TASKS" audio_command_only; } \
&& ! csv_has "$SELECTED_SOURCES" qwen3_tts_seed_audio_command; then
echo "选择了 audio_command 任务,但没有选择 qwen3_tts_seed_audio_command" >&2
exit 2
fi
if [[ "$OUTPUT_KIND" == "hidden" ]] && csv_has "$SELECTED_SOURCES" distilled_video_0226_clip50; then
echo "distilled_video_0226_clip50 尚未接入正式训练配置,hidden 模式拒绝静默生成无消费者的数据。" >&2
exit 2
fi
if [[ "$HIDDEN_LAYOUT" == "source" ]]; then
if [[ "$OUTPUT_KIND" != "hidden" \
|| "$SELECTED_TASKS" != "audio_command_text" \
|| "$SELECTED_SOURCES" != "qwen3_tts_seed_audio_command" ]]; then
echo "source 布局只用于 HFD 的 qwen3_tts_seed_audio_command/audio_command_text。" >&2
echo "其他多任务组合请显式设置 PSI_PREP_HIDDEN_LAYOUT=oss。" >&2
exit 2
fi
if [[ "$AUDIO_COMMAND_FULL_AUDIO" != "0" ]]; then
echo "source 布局必须按流式窗口导出,AUDIO_COMMAND_FULL_AUDIO 必须为 0。" >&2
exit 2
fi
if [[ "$HIDDEN_BATCH_SIZE" != "1" ]]; then
echo "HFD source hidden 必须逐窗口执行真实在线 B=1 编码,HIDDEN_BATCH_SIZE 必须为 1。" >&2
exit 2
fi
if [[ "$LIMIT" != "0" \
&& "$OUTPUT_ROOT" == "/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/videogen/hfd_audio_command_source" ]]; then
echo "有限 source 测试不能写入正式目录;请同时设置 PSI_PREP_OUTPUT_ROOT 到临时目录。" >&2
exit 2
fi
if ! "$PYTHON_BIN" - "$AUDIO_COMMAND_WINDOW_SECONDS" \
"$AUDIO_COMMAND_SAMPLE_STEP_FRAMES" <<'PY'
import math, sys
valid = (
math.isclose(float(sys.argv[1]), 0.6, rel_tol=0.0, abs_tol=1e-9)
and int(sys.argv[2]) == 20
)
raise SystemExit(0 if valid else 1)
PY
then
echo "HFD source 布局固定使用 0.6 秒窗口和 20 帧/0.4 秒步长。" >&2
exit 2
fi
fi
OUTPUT_ROOT=$($PYTHON_BIN -c 'import os,sys; print(os.path.abspath(os.path.expanduser(sys.argv[1])))' "$OUTPUT_ROOT")
AUDIO_WAV_ROOT=$($PYTHON_BIN -c 'import os,sys; print(os.path.abspath(os.path.expanduser(sys.argv[1])))' "$AUDIO_WAV_ROOT")
WORK_ROOT="${OUTPUT_ROOT}.work"
if [[ "$OUTPUT_KIND" == "embeds" ]]; then
VIDEO_CACHE_ROOT="$OUTPUT_ROOT"
FINAL_VIDEO_SUPPORT_ROOT="$OUTPUT_ROOT"
else
VIDEO_CACHE_ROOT="$WORK_ROOT/video_clip_dense_stride1"
FINAL_VIDEO_SUPPORT_ROOT="$OUTPUT_ROOT/training_support/video_clip_dense_stride1"
fi
HIDDEN_STAGING_ROOT="$WORK_ROOT/hidden_source_shards"
if [[ "$HIDDEN_LAYOUT" == "source" ]]; then
HIDDEN_EXPORT_ROOT="$OUTPUT_ROOT"
else
HIDDEN_EXPORT_ROOT="$HIDDEN_STAGING_ROOT"
fi
SOURCE_CONFIG_FILE="$WORK_ROOT/source_presets.json"
SOURCE_SPLIT_ROOT="$OUTPUT_ROOT/training_support/source_split_v1"
INDEX_TMP_DIR="/tmp/psi0_unified_preprocess_indexes.$$"
DANCE_AUDIO_WAV_ROOT="$AUDIO_WAV_ROOT/psi0_dance_audio"
AUDIO_2264_WAV_ROOT="$AUDIO_WAV_ROOT/2264_with_audio"
video_task_selected=0
if csv_has "$SELECTED_TASKS" video_clip_text || csv_has "$SELECTED_TASKS" video_clip_only; then
video_task_selected=1
fi
VIDEO_SOURCES=()
for source in dance_1723_video_10s_clip50 motion_example_0506_demo_clip50 2264_with_audio_clip50 distilled_video_0226_clip50; do
if csv_has "$SELECTED_SOURCES" "$source"; then
VIDEO_SOURCES+=("$source")
fi
done
if [[ "$OUTPUT_KIND" == "embeds" && ${#VIDEO_SOURCES[@]} -eq 0 ]]; then
echo "embeds 模式至少要选择一个 video_clip 数据源" >&2
exit 2
fi
if [[ $video_task_selected -eq 1 && ${#VIDEO_SOURCES[@]} -eq 0 ]]; then
echo "选择了 video_clip 任务,但没有选择 video_clip 数据源" >&2
exit 2
fi
echo "================================================================================"
echo "Psi0 统一离线预处理"
echo "================================================================================"
echo "输出类型: $OUTPUT_KIND"
echo "输出目录: $OUTPUT_ROOT"
echo "hidden 布局: $HIDDEN_LAYOUT"
echo "WAV 输入目录: $AUDIO_WAV_ROOT"
echo "任务: $SELECTED_TASKS"
echo "数据源: $SELECTED_SOURCES"
echo "video 工作目录: $VIDEO_CACHE_ROOT"
echo "hidden 写入: $HIDDEN_EXPORT_ROOT"
echo "torchrun 地址: $TORCHRUN_LOCAL_ADDR"
echo "audio 帧范围: $AUDIO_FRAME_COUNT_SOURCE"
echo "audio 窗口: ${AUDIO_WINDOW_SECONDS}s,stride=${AUDIO_SAMPLE_STEP_FRAMES} motion frame(s)"
echo "audio-command: full_audio=$AUDIO_COMMAND_FULL_AUDIO,${AUDIO_COMMAND_WINDOW_SECONDS}s,stride=${AUDIO_COMMAND_SAMPLE_STEP_FRAMES} motion frame(s)"
echo "Qwen 编码契约: batch=$HIDDEN_BATCH_SIZE,bf16,thinker 最后一层 hidden"
echo "在线精确复核: ${ONLINE_VERIFY_SOURCES} 个完整 WAV,要求逐元素完全一致"
echo "source split: $BUILD_SOURCE_SPLIT"
echo "测试条数限制: $LIMIT"
echo "================================================================================"
if [[ "$DRY_RUN" == "1" ]]; then
echo "[dry-run] 配置检查通过,不写文件。"
exit 0
fi
export PYTHONPATH="$PROJECT_ROOT/src${PYTHONPATH:+:$PYTHONPATH}"
export PYTHONUNBUFFERED=1
export TOKENIZERS_PARALLELISM=false
export OMP_NUM_THREADS=4
export MKL_NUM_THREADS=1
export OPENBLAS_NUM_THREADS=1
export NUMEXPR_NUM_THREADS=1
ulimit -n 1048576 2>/dev/null || true
mkdir -p "$OUTPUT_ROOT" "$WORK_ROOT" "$INDEX_TMP_DIR"
$PYTHON_BIN - "$WORK_ROOT/.psi0_preprocess_work.json" "$OUTPUT_ROOT" <<'PY'
import json, os, sys
from pathlib import Path
path = Path(sys.argv[1])
payload = {"format": "psi0_preprocess_work_v1", "output_root": str(Path(sys.argv[2]).resolve())}
tmp = path.with_name(f".{path.name}.{os.getpid()}.tmp")
tmp.write_text(json.dumps(payload, indent=2), encoding="utf-8")
os.replace(tmp, path)
PY
prepare_audio_wav_source() {
local source=$1
local dataset_root metadata_jsonl output_root
case "$source" in
psi0_dance_audio)
dataset_root="$DANCE_AV_ROOT"
metadata_jsonl="$DANCE_AV_ROOT/metadata_audio_npy.jsonl"
output_root="$DANCE_AUDIO_WAV_ROOT"
;;
2264_with_audio)
dataset_root="$AUDIO_2264_ROOT"
metadata_jsonl="$AUDIO_2264_ROOT/metadata_audio_npy.jsonl"
output_root="$AUDIO_2264_WAV_ROOT"
;;
*)
echo "未知 WAV audio source: $source" >&2
return 2
;;
esac
local -a args=(
scripts/data/materialize_audio_wav.py
--dataset-root "$dataset_root"
--metadata-jsonl "$metadata_jsonl"
--output-root "$output_root"
--output-metadata-jsonl metadata.jsonl
--input-kind auto
--input-path-key audio_path
--output-path-key audio_path
--absolute-path-keys converted_path,motion_dir,motion_path
--strip-input-prefix audio_npy_yth
--output-audio-dir audio
--sample-rate 16000
--num-workers "$AUDIO_WAV_WORKERS"
--skip-media-without-audio
)
if [[ "$LIMIT" != "0" ]]; then
args+=(--limit "$LIMIT")
fi
if [[ "$OVERWRITE" == "1" ]]; then
args+=(--overwrite)
fi
echo "[audio wav] source=$source output=$output_root"
"$PYTHON_BIN" -u "${args[@]}"
}
if csv_has "$SELECTED_TASKS" audio_text || csv_has "$SELECTED_TASKS" audio_only; then
csv_has "$SELECTED_SOURCES" psi0_dance_audio && prepare_audio_wav_source psi0_dance_audio
csv_has "$SELECTED_SOURCES" 2264_with_audio && prepare_audio_wav_source 2264_with_audio
fi
if csv_has "$SELECTED_TASKS" audio_command_text || csv_has "$SELECTED_TASKS" audio_command_only; then
[[ -s "$QWEN3_TTS_ROOT/metadata.jsonl" && -d "$QWEN3_TTS_ROOT/seed_audio" ]] || {
echo "Qwen3-TTS 原始 WAV 或 metadata.jsonl 不存在: $QWEN3_TTS_ROOT" >&2
exit 1
}
fi
common_video_args() {
VIDEO_ARGS=(
--clip-frames 50
--action-chunk-frames 30
--chunk-stride-frames 1
--motion-fps 50
--clips-per-video 1
--clip-selection-mode all
--random-seed 42
--middle-fraction 0.8
--num-workers "$VIDEO_BUILD_WORKERS"
--chunk-workers 1
--metadata-only
--shard-num "$VIDEO_METADATA_SHARDS"
)
if [[ "$LIMIT" != "0" ]]; then
VIDEO_ARGS+=(--limit "$LIMIT")
fi
if [[ "$OVERWRITE" == "1" ]]; then
VIDEO_ARGS+=(--overwrite)
fi
}
video_output_dir() {
case "$1" in
dance_1723_video_10s_clip50) printf '%s/dance_1723_video_10s' "$VIDEO_CACHE_ROOT" ;;
motion_example_0506_demo_clip50) printf '%s/motion_example_0506_demo' "$VIDEO_CACHE_ROOT" ;;
2264_with_audio_clip50) printf '%s/2264_with_audio' "$VIDEO_CACHE_ROOT" ;;
distilled_video_0226_clip50) printf '%s/distilled_video_0226' "$VIDEO_CACHE_ROOT" ;;
*) echo "未知 video source: $1" >&2; return 2 ;;
esac
}
build_video_metadata() {
local source=$1
local output_dir
output_dir=$(video_output_dir "$source")
local shard_id
for ((shard_id=0; shard_id<VIDEO_METADATA_SHARDS; shard_id++)); do
common_video_args
VIDEO_ARGS+=(--output-root "$output_dir" --shard-id "$shard_id")
case "$source" in
dance_1723_video_10s_clip50)
VIDEO_ARGS+=(
--dataset dance
--dance-motion-root "$DANCE_MOTION_ROOT"
--dance-video-root "$DANCE_AV_ROOT"
)
;;
motion_example_0506_demo_clip50)
VIDEO_ARGS+=(
--dataset motion_example
--motion-example-root "$MOTION_EXAMPLE_ROOT"
--metadata-jsonl metadata.jsonl
)
;;
2264_with_audio_clip50)
VIDEO_ARGS+=(
--dataset manifest
--manifest-root "$AUDIO_2264_ROOT"
--metadata-jsonl metadata.jsonl
--motion-metadata-jsonl metadata_convert.jsonl
--motion-type-default 2264_with_audio
--rel-name-style name
)
;;
distilled_video_0226_clip50)
VIDEO_ARGS+=(
--dataset manifest
--manifest-root "$DISTILLED_VIDEO_ROOT"
--metadata-jsonl metadata.jsonl
--motion-metadata-jsonl metadata_convert.jsonl
--motion-type-default distilled_video_0226
--rel-name-style name
)
;;
esac
echo "[video metadata] source=$source shard=$shard_id/$VIDEO_METADATA_SHARDS"
"$PYTHON_BIN" scripts/data/build_video_clip_dataset.py "${VIDEO_ARGS[@]}"
done
[[ -s "$output_dir/metadata.jsonl" ]] || {
echo "video metadata 未生成: $output_dir/metadata.jsonl" >&2
return 1
}
}
cache_video_embeds() {
local source=$1
local output_dir
output_dir=$(video_output_dir "$source")
local shard_id metadata_name
for ((shard_id=0; shard_id<VIDEO_METADATA_SHARDS; shard_id++)); do
if [[ "$VIDEO_METADATA_SHARDS" == "1" ]]; then
metadata_name=metadata.jsonl
else
printf -v metadata_name 'metadata.shard-%05d-of-%05d.jsonl' "$shard_id" "$VIDEO_METADATA_SHARDS"
fi
EMBED_ARGS=(
--dataset-root "$output_dir"
--metadata-jsonl "$metadata_name"
--embed-root "$output_dir/embeds"
--model-path "$MODEL_PATH"
--video-sample-fps "$VIDEO_SAMPLE_FPS"
--video-size-pixels "$VIDEO_SIZE_PIXELS"
--video-decoder "$VIDEO_DECODER"
--dtype float16
--num-workers "$VIDEO_EMBED_WORKERS"
--existing-cache-check size
--shard-num "$VIDEO_METADATA_SHARDS"
--shard-id "$shard_id"
--input-already-sharded
--direct-source-clips
)
if [[ "$LIMIT" != "0" ]]; then
EMBED_ARGS+=(--limit "$LIMIT")
fi
if [[ "$OVERWRITE" == "1" ]]; then
EMBED_ARGS+=(--overwrite)
fi
echo "[video embeds] source=$source shard=$shard_id/$VIDEO_METADATA_SHARDS"
"$PYTHON_BIN" "$PROJECT_ROOT/backup/cache_video_omni_embeds.py" "${EMBED_ARGS[@]}"
done
[[ -s "$output_dir/embeds/manifest.json" ]] || {
echo "video embed manifest 未生成: $output_dir/embeds/manifest.json" >&2
return 1
}
}
if [[ $video_task_selected -eq 1 || "$OUTPUT_KIND" == "embeds" ]]; then
for source in "${VIDEO_SOURCES[@]}"; do
build_video_metadata "$source"
cache_video_embeds "$source"
done
fi
write_source_config() {
export PSI_PREP_SELECTED_SOURCES="$SELECTED_SOURCES"
export PSI_PREP_VIDEO_CACHE_ROOT="$VIDEO_CACHE_ROOT"
export PSI_PREP_SEED_ROOT="$SEED_ROOT"
export PSI_PREP_SEED_METADATA="$SEED_METADATA"
export PSI_PREP_QWEN3_TTS_ROOT="$QWEN3_TTS_ROOT"
export PSI_PREP_DANCE_AUDIO_WAV_ROOT="$DANCE_AUDIO_WAV_ROOT"
export PSI_PREP_2264_AUDIO_WAV_ROOT="$AUDIO_2264_WAV_ROOT"
"$PYTHON_BIN" - "$SOURCE_CONFIG_FILE" <<'PY'
import json, os, sys
from pathlib import Path
selected = {value for value in os.environ["PSI_PREP_SELECTED_SOURCES"].split(",") if value}
video_root = Path(os.environ["PSI_PREP_VIDEO_CACHE_ROOT"])
seed_root = Path(os.environ["PSI_PREP_SEED_ROOT"])
qwen_root = Path(os.environ["PSI_PREP_QWEN3_TTS_ROOT"])
dance_wav_root = Path(os.environ["PSI_PREP_DANCE_AUDIO_WAV_ROOT"])
root_2264_wav = Path(os.environ["PSI_PREP_2264_AUDIO_WAV_ROOT"])
sources = [
{
"name": "seed_text", "modality": "text", "dataset_root": str(seed_root),
"manifest_path": os.environ["PSI_PREP_SEED_METADATA"], "allowed_tasks": ["text"],
},
{
"name": "2264_with_audio", "modality": "audio", "dataset_root": str(root_2264_wav),
"manifest_path": str(root_2264_wav / "metadata.jsonl"),
"allowed_tasks": ["audio_text", "audio_only"],
"audio_file_extensions": [".wav"],
},
{
"name": "psi0_dance_audio", "modality": "audio", "dataset_root": str(dance_wav_root),
"manifest_path": str(dance_wav_root / "metadata.jsonl"),
"allowed_tasks": ["audio_text", "audio_only"],
"audio_file_extensions": [".wav"],
},
{
"name": "qwen3_tts_seed_audio_command", "modality": "audio", "dataset_root": str(qwen_root),
"manifest_path": str(qwen_root / "metadata.jsonl"),
"allowed_tasks": ["audio_command_text", "audio_command_only"],
"instruction_key": "content_short_description", "is_audio_dir_manifest": True,
"audio_file_extensions": [".wav"],
},
]
for name, directory in (
("dance_1723_video_10s_clip50", "dance_1723_video_10s"),
("motion_example_0506_demo_clip50", "motion_example_0506_demo"),
("2264_with_audio_clip50", "2264_with_audio"),
("distilled_video_0226_clip50", "distilled_video_0226"),
):
root = video_root / directory
sources.append({
"name": name, "modality": "video_clip", "dataset_root": str(root),
"manifest_path": str(root / "metadata.jsonl"),
"allowed_tasks": ["video_clip_text", "video_clip_only"],
"instruction_key": "video_caption", "video_embed_dir": str(root / "embeds"),
})
sources = [source for source in sources if source["name"] in selected]
path = Path(sys.argv[1])
path.parent.mkdir(parents=True, exist_ok=True)
tmp = path.with_name(f".{path.name}.{os.getpid()}.tmp")
tmp.write_text(json.dumps({"sources": sources}, ensure_ascii=False, indent=2), encoding="utf-8")
os.replace(tmp, path)
PY
}
monitor_stopped=0
restore_gpu_monitor() {
if [[ "$monitor_stopped" == "1" && "$MANAGE_LOCAL_GPU_MONITOR" == "1" && -f "$GPU_MONITOR_SCRIPT" ]]; then
bash "$GPU_MONITOR_SCRIPT"
monitor_stopped=0
fi
}
trap restore_gpu_monitor EXIT
stop_local_gpu_monitor() {
if [[ "$MANAGE_LOCAL_GPU_MONITOR" == "1" ]]; then
local monitor_dir=${GPU_MONITOR_SCRIPT%/*}
pkill -f "$monitor_dir/[m]onitor.sh" 2>/dev/null || true
pkill -f "$monitor_dir/[m]onitor.py" 2>/dev/null || true
monitor_stopped=1
echo "[GPU] 已停止本项目占卡 monitor;不检查、不阻止其他 GPU 进程。"
fi
}
if [[ "$OUTPUT_KIND" == "hidden" ]]; then
write_source_config
stop_local_gpu_monitor
export CUDA_VISIBLE_DEVICES
export NCCL_DEBUG=WARN
export PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True
VISIBLE_GPU_COUNT=$(
"$PYTHON_BIN" - <<'PY'
import torch
print(torch.cuda.device_count())
PY
)
if [[ ! "$VISIBLE_GPU_COUNT" =~ ^[1-9][0-9]*$ ]]; then
echo "PyTorch 没有识别到可用 CUDA GPU(device_count=$VISIBLE_GPU_COUNT)。" >&2
exit 2
fi
NPROC=$(( VISIBLE_GPU_COUNT * WORKERS_PER_GPU ))
if [[ "$HIDDEN_LAYOUT" == "source" ]]; then
HIDDEN_SAVE_WORKERS=4
HIDDEN_MAX_PENDING_SAVES=8
else
HIDDEN_SAVE_WORKERS=16
HIDDEN_MAX_PENDING_SAVES=32
fi
HIDDEN_ARGS=(
scripts/data/cache_omni_hidden_states.py
--mode export
--model-path "$MODEL_PATH"
--output-root "$HIDDEN_EXPORT_ROOT"
--tasks "$SELECTED_TASKS"
--sources "$SELECTED_SOURCES"
--source-config-file "$SOURCE_CONFIG_FILE"
--motion-fps 50
--audio-sample-step-frames "$AUDIO_SAMPLE_STEP_FRAMES"
--audio-frame-count-source "$AUDIO_FRAME_COUNT_SOURCE"
--audio-window-seconds "$AUDIO_WINDOW_SECONDS"
--audio-left-context-seconds 0.0
--audio-command-sample-step-frames "$AUDIO_COMMAND_SAMPLE_STEP_FRAMES"
--audio-command-window-seconds "$AUDIO_COMMAND_WINDOW_SECONDS"
--audio-sampling-rate 16000
--audio-batch-size "$HIDDEN_BATCH_SIZE"
--text-batch-size "$HIDDEN_BATCH_SIZE"
--video-batch-size "$HIDDEN_BATCH_SIZE"
--num-io-workers 8
--num-save-workers "$HIDDEN_SAVE_WORKERS"
--max-pending-save-batches "$HIDDEN_MAX_PENDING_SAVES"
--save-dtype bf16
--storage-format "$HIDDEN_LAYOUT"
)
if [[ "$HIDDEN_LAYOUT" == "oss" ]]; then
HIDDEN_ARGS+=(--shard-size "$HIDDEN_SHARD_SIZE")
fi
if [[ $video_task_selected -eq 1 ]]; then
HIDDEN_ARGS+=(--confirm-full-video)
fi
if [[ "$AUDIO_COMMAND_FULL_AUDIO" == "1" ]]; then
HIDDEN_ARGS+=(--audio-command-full-audio)
else
HIDDEN_ARGS+=(--no-audio-command-full-audio)
fi
if [[ "$LIMIT" != "0" ]]; then
if [[ "$HIDDEN_LAYOUT" == "source" ]]; then
HIDDEN_ARGS+=(--limit-sources "$LIMIT")
else
HIDDEN_ARGS+=(--limit "$LIMIT")
fi
fi
if [[ "$OVERWRITE" == "1" ]]; then
HIDDEN_ARGS+=(--overwrite)
fi
echo "[hidden export] requested_GPUs=$CUDA_VISIBLE_DEVICES detected_GPUs=$VISIBLE_GPU_COUNT workers_per_gpu=$WORKERS_PER_GPU nproc=$NPROC"
"$PYTHON_BIN" -m torch.distributed.run --standalone \
--local_addr="$TORCHRUN_LOCAL_ADDR" --nproc_per_node="$NPROC" \
"${HIDDEN_ARGS[@]}"
if [[ "$HIDDEN_LAYOUT" == "source" ]]; then
echo "[hidden verify] 用真实在线 B=1 Qwen 路径复算 ${ONLINE_VERIFY_SOURCES} 个完整 WAV"
"$PYTHON_BIN" -u scripts/data/cache_omni_hidden_states.py \
--mode verify \
--model-path "$MODEL_PATH" \
--output-root "$HIDDEN_EXPORT_ROOT" \
--tasks "$SELECTED_TASKS" \
--sources "$SELECTED_SOURCES" \
--source-config-file "$SOURCE_CONFIG_FILE" \
--motion-fps 50 \
--audio-frame-count-source "$AUDIO_FRAME_COUNT_SOURCE" \
--audio-command-sample-step-frames "$AUDIO_COMMAND_SAMPLE_STEP_FRAMES" \
--audio-command-window-seconds "$AUDIO_COMMAND_WINDOW_SECONDS" \
--audio-sampling-rate 16000 \
--audio-batch-size 1 \
--save-dtype bf16 \
--storage-format source \
--verify-sources "$ONLINE_VERIFY_SOURCES" \
--no-audio-command-full-audio
fi
restore_gpu_monitor
if [[ "$HIDDEN_LAYOUT" == "oss" ]]; then
MERGE_ARGS=(
scripts/data/merge_omni_hidden_for_oss.py
--src-root "$HIDDEN_STAGING_ROOT"
--dst-root "$OUTPUT_ROOT"
--group-by task
--samples-per-shard "$FINAL_SAMPLES_PER_SHARD"
--max-shard-gib "$FINAL_MAX_SHARD_GIB"
--workers "$MERGE_WORKERS"
--max-pending "$((MERGE_WORKERS * 2))"
--executor process
--start-method fork
--seed "$SHUFFLE_SEED"
--verify-samples "$VERIFY_SAMPLES"
--source-reader-cache-size "$MERGE_SOURCE_READER_CACHE"
--index-tmp-dir "$INDEX_TMP_DIR"
--write-compat-indexes
--no-write-jsonl-index
)
IFS=',' read -r -a task_list <<< "$SELECTED_TASKS"
for task in "${task_list[@]}"; do
MERGE_ARGS+=(--include-task "$task")
done
if [[ "$OVERWRITE" == "1" ]]; then
MERGE_ARGS+=(--overwrite)
fi
"$PYTHON_BIN" -u "${MERGE_ARGS[@]}"
if [[ "$VERIFY_CANONICAL_COUNTS" == "1" && "$LIMIT" == "0" ]]; then
"$PYTHON_BIN" - "$OUTPUT_ROOT" "$SELECTED_TASKS" "$SELECTED_SOURCES" "$AUDIO_FRAME_COUNT_SOURCE" \
"$AUDIO_COMMAND_FULL_AUDIO" "$AUDIO_COMMAND_SAMPLE_STEP_FRAMES" "$AUDIO_COMMAND_WINDOW_SECONDS" <<'PY'
import json
import sys
from pathlib import Path
root = Path(sys.argv[1])
tasks = {value for value in sys.argv[2].split(",") if value}
sources = {value for value in sys.argv[3].split(",") if value}
frame_source = sys.argv[4]
command_full_audio = sys.argv[5] == "1"
command_step_frames = int(sys.argv[6])
command_window_seconds = float(sys.argv[7])
canonical_tasks = {"text", "audio_text", "audio_command_text", "video_clip_text"}
canonical_sources = {
"seed_text", "psi0_dance_audio", "2264_with_audio",
"qwen3_tts_seed_audio_command", "dance_1723_video_10s_clip50",
"motion_example_0506_demo_clip50", "2264_with_audio_clip50",
}
is_windowed_command = (
not command_full_audio
and command_step_frames == 20
and command_window_seconds == 0.6
)
if (
tasks == canonical_tasks
and sources == canonical_sources
and frame_source == "metadata"
and is_windowed_command
):
expected = {
"text": 142_220,
"audio_text": 14_812_682,
"audio_command_text": 304_986,
"video_clip_text": 18_558_588,
}
elif (
tasks == {"audio_command_text"}
and sources == {"qwen3_tts_seed_audio_command"}
and is_windowed_command
):
expected = {"audio_command_text": 304_986}
else:
expected = {}
print("[canonical counts] 非默认组合,跳过固定条数检查。")
if expected:
for task, expected_count in expected.items():
marker = root / "tasks" / task / ".merge_complete.json"
payload = json.loads(marker.read_text(encoding="utf-8"))
actual = int(payload.get("samples", -1))
if actual != expected_count:
raise RuntimeError(
f"Canonical sample count mismatch for {task}: {actual} != {expected_count}"
)
print(f"[canonical counts] {task}: {actual:,}(符合当前默认口径)")
PY
fi
else
"$PYTHON_BIN" - "$OUTPUT_ROOT" "$NPROC" "$LIMIT" <<'PY'
import sqlite3
import sys
from pathlib import Path
root = Path(sys.argv[1])
world_size = int(sys.argv[2])
limit_sources = int(sys.argv[3])
source_dir = (
root
/ "audio"
/ "qwen3_tts_seed_audio_command"
/ "audio_command_text"
/ "sources"
)
indexes = sorted(source_dir.glob(f"index.ws{world_size}.rank*.sqlite"))
if len(indexes) != world_size:
raise RuntimeError(
f"Source index count mismatch: {len(indexes)} != {world_size} under {source_dir}"
)
rows = {}
for index_path in indexes:
with sqlite3.connect(f"file:{index_path}?mode=ro", uri=True) as conn:
for row in conn.execute(
"SELECT key, path, count, window_seconds, step_frames, "
"motion_fps, sampling_rate FROM sources"
):
key = str(row[0])
value = (
str(row[1]),
int(row[2]),
float(row[3]),
int(row[4]),
int(row[5]),
int(row[6]),
)
previous = rows.get(key)
if previous is not None and previous != value:
raise RuntimeError(f"Conflicting source row for {key}")
rows[key] = value
window_total = 0
for key, (relative_path, count, window_seconds, step_frames, fps, rate) in rows.items():
if (
count <= 0
or window_seconds != 0.6
or step_frames != 20
or fps != 50
or rate != 16000
):
raise RuntimeError(
f"Invalid HFD timing/index row for {key}: "
f"{count}, {window_seconds}, {step_frames}, {fps}, {rate}"
)
path = source_dir / relative_path
if not path.is_file():
raise FileNotFoundError(path)
window_total += count
if limit_sources == 0:
if len(rows) != 41_694:
raise RuntimeError(f"HFD source count mismatch: {len(rows):,} != 41,694")
if window_total != 304_986:
raise RuntimeError(
f"HFD window count mismatch: {window_total:,} != 304,986"
)
print(
f"[source verify] sources={len(rows):,} windows={window_total:,} "
f"indexes={len(indexes)} timing=0.6s/0.4s"
)
PY
fi
fi
copy_hidden_training_metadata() {
[[ "$OUTPUT_KIND" == "hidden" ]] || return 0
[[ $video_task_selected -eq 1 ]] || return 0
local source src_dir dst_dir
for source in "${VIDEO_SOURCES[@]}"; do
src_dir=$(video_output_dir "$source")
case "$source" in
dance_1723_video_10s_clip50) dst_dir="$FINAL_VIDEO_SUPPORT_ROOT/dance_1723_video_10s" ;;
motion_example_0506_demo_clip50) dst_dir="$FINAL_VIDEO_SUPPORT_ROOT/motion_example_0506_demo" ;;
2264_with_audio_clip50) dst_dir="$FINAL_VIDEO_SUPPORT_ROOT/2264_with_audio" ;;
distilled_video_0226_clip50) dst_dir="$FINAL_VIDEO_SUPPORT_ROOT/distilled_video_0226" ;;
esac
"$PYTHON_BIN" - "$src_dir/metadata.jsonl" "$dst_dir/metadata.jsonl" <<'PY'
import os, shutil, sys
from pathlib import Path
src, dst = map(Path, sys.argv[1:3])
dst.parent.mkdir(parents=True, exist_ok=True)
tmp = dst.with_name(f".{dst.name}.{os.getpid()}.tmp")
try:
try:
os.link(src, tmp)
except OSError:
shutil.copyfile(src, tmp)
os.replace(tmp, dst)
finally:
try: tmp.unlink()
except FileNotFoundError: pass
PY
done
}
if [[ $video_task_selected -eq 1 ]]; then
copy_hidden_training_metadata
fi
write_support_config() {
local support_config=$1
export PSI_PREP_SELECTED_SOURCES="$SELECTED_SOURCES"
export PSI_PREP_SEED_ROOT="$SEED_ROOT"
export PSI_PREP_QWEN3_TTS_ROOT="$QWEN3_TTS_ROOT"
export PSI_PREP_SOURCE_SPLIT_ROOT="$SOURCE_SPLIT_ROOT"
export PSI_PREP_FINAL_VIDEO_ROOT="$FINAL_VIDEO_SUPPORT_ROOT"
export PSI_PREP_DANCE_AUDIO_WAV_ROOT="$DANCE_AUDIO_WAV_ROOT"
export PSI_PREP_2264_AUDIO_WAV_ROOT="$AUDIO_2264_WAV_ROOT"
export PSI_PREP_AUDIO_COMMAND_FULL_AUDIO="$AUDIO_COMMAND_FULL_AUDIO"
export PSI_PREP_AUDIO_COMMAND_SAMPLE_STEP_FRAMES="$AUDIO_COMMAND_SAMPLE_STEP_FRAMES"
export PSI_PREP_AUDIO_COMMAND_WINDOW_SECONDS="$AUDIO_COMMAND_WINDOW_SECONDS"
"$PYTHON_BIN" - "$support_config" <<'PY'
import json, os, sys
from pathlib import Path
selected = {value for value in os.environ["PSI_PREP_SELECTED_SOURCES"].split(",") if value}
split_root = Path(os.environ["PSI_PREP_SOURCE_SPLIT_ROOT"])
video_root = Path(os.environ["PSI_PREP_FINAL_VIDEO_ROOT"])
qwen_root = Path(os.environ["PSI_PREP_QWEN3_TTS_ROOT"])
seed_root = Path(os.environ["PSI_PREP_SEED_ROOT"])
dance_wav_root = Path(os.environ["PSI_PREP_DANCE_AUDIO_WAV_ROOT"])
root_2264_wav = Path(os.environ["PSI_PREP_2264_AUDIO_WAV_ROOT"])
audio_command_full_audio = os.environ["PSI_PREP_AUDIO_COMMAND_FULL_AUDIO"] == "1"
audio_command_step_frames = int(os.environ["PSI_PREP_AUDIO_COMMAND_SAMPLE_STEP_FRAMES"])
audio_command_window_seconds = float(os.environ["PSI_PREP_AUDIO_COMMAND_WINDOW_SECONDS"])
audio = []
if "psi0_dance_audio" in selected:
audio.append({
"name": "psi0_dance_audio", "split_source_namespace": "dance_1723_video_10s",
"dataset_root": str(dance_wav_root), "metadata_jsonl": str(dance_wav_root / "metadata.jsonl"),
"motion_dir_key": "converted_path", "audio_path_key": "audio_path",
"audio_file_extensions": ".wav",
"records_cache_dir": str(split_root / "audio_records"), "dataset_name": "seed-audio-g1",
})
if "2264_with_audio" in selected:
audio.append({
"name": "2264_with_audio", "split_source_namespace": "2264_with_audio",
"dataset_root": str(root_2264_wav), "metadata_jsonl": str(root_2264_wav / "metadata.jsonl"),
"motion_dir_key": "converted_path", "audio_path_key": "audio_path",
"audio_file_extensions": ".wav",
"records_cache_dir": str(split_root / "audio_records"), "dataset_name": "2264-with-audio-g1",
})
audio_command = []
if "qwen3_tts_seed_audio_command" in selected:
audio_command.append({
"name": "qwen3_tts_seed_audio_command", "split_source_namespace": "seed",
"root_dir": str(seed_root), "dataset_root": str(qwen_root),
"metadata_jsonl": str(qwen_root / "metadata.jsonl"),
"motion_dir_key": "filename", "audio_path_key": "audio_path",
"instruction_key": "content_short_description",
"motion_metadata_csv": str(seed_root / "metadata/seed_metadata_v004.csv"),
"motion_metadata_key": "filename", "motion_metadata_path_key": "move_g1_path",
"audio_file_extensions": ".wav", "records_cache_dir": str(split_root / "audio_command_records"),
"audio_task": "command", "dataset_name": "seed-audio-command-g1",
"full_audio": audio_command_full_audio,
"sample_step_frames": audio_command_step_frames,
"audio_window_seconds": audio_command_window_seconds,
})
video = []
for source_name, namespace, directory, dataset_name in (
("dance_1723_video_10s_clip50", "dance_1723_video_10s", "dance_1723_video_10s", "dance-video-clip-g1"),
("motion_example_0506_demo_clip50", "motion_example_0506_demo", "motion_example_0506_demo", "motion-example-video-clip-g1"),
("2264_with_audio_clip50", "2264_with_audio", "2264_with_audio", "2264-with-audio-video-clip-g1"),
):
if source_name not in selected:
continue
root = video_root / directory
video.append({
"name": source_name, "split_source_namespace": namespace,
"dataset_root": str(root), "metadata_jsonl": "metadata.jsonl",
"motion_dir_key": "converted_path", "video_path_key": "video_path",
"instruction_key": "video_caption", "video_embed_dir": str(root / "embeds"),
"records_cache_dir": str(split_root / "video_clip_records"), "dataset_name": dataset_name,
})
payload = {"audio": audio, "audio_command": audio_command, "video_clip": video}
path = Path(sys.argv[1])
path.parent.mkdir(parents=True, exist_ok=True)
tmp = path.with_name(f".{path.name}.{os.getpid()}.tmp")
tmp.write_text(json.dumps(payload, ensure_ascii=False, indent=2), encoding="utf-8")
os.replace(tmp, path)
PY
}
if [[ "$BUILD_SOURCE_SPLIT" == "1" ]]; then
SUPPORT_CONFIG_FILE="$WORK_ROOT/source_split_sources.json"
mkdir -p "$SOURCE_SPLIT_ROOT/runtime_indexes"
write_support_config "$SUPPORT_CONFIG_FILE"
AUDIO_SOURCES_JSON=$($PYTHON_BIN -c 'import json,sys; print(json.dumps(json.load(open(sys.argv[1]))["audio"],separators=(",",":")))' "$SUPPORT_CONFIG_FILE")
AUDIO_COMMAND_SOURCES_JSON=$($PYTHON_BIN -c 'import json,sys; print(json.dumps(json.load(open(sys.argv[1]))["audio_command"],separators=(",",":")))' "$SUPPORT_CONFIG_FILE")
VIDEO_CLIP_SOURCES_JSON=$($PYTHON_BIN -c 'import json,sys; print(json.dumps(json.load(open(sys.argv[1]))["video_clip"],separators=(",",":")))' "$SUPPORT_CONFIG_FILE")
PREBUILD_TASKS=()
csv_has "$SELECTED_TASKS" text && csv_has "$SELECTED_SOURCES" seed_text && PREBUILD_TASKS+=(text)
if csv_has "$SELECTED_TASKS" audio_text || csv_has "$SELECTED_TASKS" audio_only; then PREBUILD_TASKS+=(audio); fi
if csv_has "$SELECTED_TASKS" audio_command_text || csv_has "$SELECTED_TASKS" audio_command_only; then PREBUILD_TASKS+=(audio_command); fi
[[ $video_task_selected -eq 1 ]] && PREBUILD_TASKS+=(video_clip)
PREBUILD_TASKS_CSV=$(IFS=,; echo "${PREBUILD_TASKS[*]}")
export PSI_OMNI_OSS_RUNTIME_INDEX_DIR="$SOURCE_SPLIT_ROOT/runtime_indexes"
if [[ "$OUTPUT_KIND" == "hidden" ]]; then
export PSI_PREBUILD_OMNI_OSS_KEY_INDEX=1
else
# embeds 模式没有 hidden SQLite;这里只生成训练/验证 source split 记录。
export PSI_PREBUILD_OMNI_OSS_KEY_INDEX=0
fi
export PSI_VIDEO_UNIQUE_RECORD_CACHE_FALLBACK=0
if [[ "$AUDIO_COMMAND_FULL_AUDIO" == "1" ]]; then
AUDIO_COMMAND_FULL_AUDIO_ARG=--audio-command-full-audio
else
AUDIO_COMMAND_FULL_AUDIO_ARG=--no-audio-command-full-audio
fi
"$PYTHON_BIN" -u scripts/data/prebuild_multimodal_records_cache.py \
--tasks "$PREBUILD_TASKS_CSV" --splits train,val \
--action-chunk-size 30 --num-past-frames 0 --omni-hidden-root "$OUTPUT_ROOT" \
--text-root-dir "$SEED_ROOT" --text-metadata-csv metadata/seed_metadata_v004.csv \
--text-records-cache-dir "$SOURCE_SPLIT_ROOT/text_records" --text-split-source-namespace seed \
--audio-sources-json "$AUDIO_SOURCES_JSON" --audio-records-cache-dir "$SOURCE_SPLIT_ROOT/audio_records" \
--audio-sample-stride 1 --audio-sample-step-frames "$AUDIO_SAMPLE_STEP_FRAMES" \
--audio-motion-fps 50 --audio-audio-sampling-rate 16000 \
--audio-audio-window-seconds "$AUDIO_WINDOW_SECONDS" \
--audio-audio-left-context-seconds 0.0 --no-audio-full-audio \
--audio-command-sources-json "$AUDIO_COMMAND_SOURCES_JSON" \
--audio-command-records-cache-dir "$SOURCE_SPLIT_ROOT/audio_command_records" \
--audio-command-sample-stride 1 --audio-command-sample-step-frames "$AUDIO_COMMAND_SAMPLE_STEP_FRAMES" \
--audio-command-motion-fps 50 \
--audio-command-audio-sampling-rate 16000 --audio-command-audio-window-seconds "$AUDIO_COMMAND_WINDOW_SECONDS" \
--audio-command-audio-left-context-seconds 0.0 "$AUDIO_COMMAND_FULL_AUDIO_ARG" \
--video-sources-json '[]' \
--video-clip-sources-json "$VIDEO_CLIP_SOURCES_JSON" \
--video-clip-records-cache-dir "$SOURCE_SPLIT_ROOT/video_clip_records" \
--video-clip-sample-stride 1 --video-clip-motion-fps 50 \
--video-clip-video-sample-fps "$VIDEO_SAMPLE_FPS" \
--video-clip-video-size-pixels "$VIDEO_SIZE_PIXELS" --video-clip-video-decoder "$VIDEO_DECODER" \
--video-clip-record-build-workers 32 --video-clip-record-build-chunk-size 4096 \
--video-clip-trust-metadata --video-clip-action-exec-horizon 20
if [[ "$OUTPUT_KIND" == "hidden" ]]; then
PREFLIGHT_ARGS=(
scripts/data/prepare_omni_oss_training.py
--oss-root "$OUTPUT_ROOT"
--runtime-index-dir "$SOURCE_SPLIT_ROOT/runtime_indexes"
--tasks "$SELECTED_TASKS"
--splits train,val
--val-ratio 0.1
--split-seed 42
)
for source in "${VIDEO_SOURCES[@]}"; do
case "$source" in
dance_1723_video_10s_clip50) namespace=dance_1723_video_10s; directory=dance_1723_video_10s ;;
motion_example_0506_demo_clip50) namespace=motion_example_0506_demo; directory=motion_example_0506_demo ;;
2264_with_audio_clip50) namespace=2264_with_audio; directory=2264_with_audio ;;
*) continue ;;
esac
PREFLIGHT_ARGS+=(--video-manifest-spec "$namespace=$FINAL_VIDEO_SUPPORT_ROOT/$directory/metadata.jsonl")
done
[[ $video_task_selected -eq 0 ]] && PREFLIGHT_ARGS+=(--skip-video-index)
"$PYTHON_BIN" -u "${PREFLIGHT_ARGS[@]}"
fi
if [[ "$LIMIT" == "0" ]] \
&& csv_has "$SELECTED_TASKS" text \
&& csv_has "$SELECTED_TASKS" audio_text \
&& csv_has "$SELECTED_TASKS" audio_command_text \
&& csv_has "$SELECTED_TASKS" video_clip_text \
&& csv_has "$SELECTED_SOURCES" seed_text \
&& csv_has "$SELECTED_SOURCES" psi0_dance_audio \
&& csv_has "$SELECTED_SOURCES" 2264_with_audio \
&& csv_has "$SELECTED_SOURCES" qwen3_tts_seed_audio_command \
&& csv_has "$SELECTED_SOURCES" dance_1723_video_10s_clip50 \
&& csv_has "$SELECTED_SOURCES" motion_example_0506_demo_clip50 \
&& csv_has "$SELECTED_SOURCES" 2264_with_audio_clip50; then
"$PYTHON_BIN" -u scripts/data/audit_source_split_artifacts.py \
--cache-root "$SOURCE_SPLIT_ROOT" \
--runtime-index-dir "$SOURCE_SPLIT_ROOT/runtime_indexes" \
--video-root "$FINAL_VIDEO_SUPPORT_ROOT"
fi
"$PYTHON_BIN" scripts/data/manage_source_split_artifacts.py build-manifest --root "$SOURCE_SPLIT_ROOT"
"$PYTHON_BIN" scripts/data/manage_source_split_artifacts.py verify --root "$SOURCE_SPLIT_ROOT"
fi
cleanup_video_resume_files() {
local source root
for source in "${VIDEO_SOURCES[@]}"; do
root=$(video_output_dir "$source")
[[ "$root" == "$VIDEO_CACHE_ROOT"/* ]] || { echo "拒绝清理意外路径: $root" >&2; return 1; }
rm -rf -- "$root/.source_done"
find "$root" -maxdepth 1 -type f \( -name 'metadata.shard-*.jsonl' -o -name '.*.tmp' \) -delete
if [[ -d "$root/embeds" ]]; then
find "$root/embeds" -maxdepth 1 -type f \( -name 'resume*.json' -o -name 'manifest.shard-*.json' -o -name '*.lock' -o -name '.*.tmp' \) -delete
fi
done
}
if [[ "$HIDDEN_LAYOUT" == "source" && "$LIMIT" != "0" ]]; then
COMPLETE_PATH="$OUTPUT_ROOT/_reports/prepare_debug_limit_${LIMIT}.json"
else
COMPLETE_PATH="$OUTPUT_ROOT/_reports/prepare_complete.json"
fi
mkdir -p "$(dirname "$COMPLETE_PATH")"
"$PYTHON_BIN" - "$COMPLETE_PATH" "$OUTPUT_KIND" "$OUTPUT_ROOT" "$SELECTED_TASKS" "$SELECTED_SOURCES" \
"$BUILD_SOURCE_SPLIT" "$MODEL_PATH" "$CUDA_VISIBLE_DEVICES" "$WORKERS_PER_GPU" \
"$HIDDEN_BATCH_SIZE" "$AUDIO_FRAME_COUNT_SOURCE" "$HIDDEN_SHARD_SIZE" \
"$FINAL_SAMPLES_PER_SHARD" "$FINAL_MAX_SHARD_GIB" "$SHUFFLE_SEED" \
"$AUDIO_WAV_ROOT" "$AUDIO_SAMPLE_STEP_FRAMES" "$AUDIO_WINDOW_SECONDS" \
"$AUDIO_COMMAND_FULL_AUDIO" "$AUDIO_COMMAND_SAMPLE_STEP_FRAMES" "$AUDIO_COMMAND_WINDOW_SECONDS" \
"$HIDDEN_LAYOUT" "$LIMIT" "$ONLINE_VERIFY_SOURCES" <<'PY'
import json, os, sys, time
from pathlib import Path
path = Path(sys.argv[1])
payload = {
"format": "psi0_unified_preprocess_v4",
"output_kind": sys.argv[2],
"output_root": str(Path(sys.argv[3]).resolve()),
"tasks": [value for value in sys.argv[4].split(",") if value],
"sources": [value for value in sys.argv[5].split(",") if value],
"source_split_ready": sys.argv[6] == "1",
"model_path": str(Path(sys.argv[7]).resolve()),
"cuda_visible_devices": sys.argv[8],
"workers_per_gpu": int(sys.argv[9]),
"hidden_batch_size": int(sys.argv[10]),
"audio_frame_count_source": sys.argv[11],
"hidden_layout": sys.argv[22],
"limit_sources": int(sys.argv[23]) if sys.argv[22] == "source" else 0,
"online_verify_sources": int(sys.argv[24]) if sys.argv[22] == "source" else 0,
"hidden_staging_shard_size": int(sys.argv[12]) if sys.argv[22] == "oss" else None,
"final_samples_per_shard": int(sys.argv[13]) if sys.argv[22] == "oss" else None,
"final_max_shard_gib": float(sys.argv[14]) if sys.argv[22] == "oss" else None,
"shuffle_seed": int(sys.argv[15]),
"audio_input_format": "wav",
"audio_wav_root": str(Path(sys.argv[16]).resolve()),
"audio_sample_step_frames": int(sys.argv[17]),
"audio_window_seconds": float(sys.argv[18]),
"audio_command_full_audio": sys.argv[19] == "1",
"audio_command_sample_step_frames": int(sys.argv[20]),
"audio_command_window_seconds": float(sys.argv[21]),
"completed_at": time.time(),
}
if sys.argv[22] == "source":
verify_path = Path(sys.argv[3]) / "_reports" / "online_hidden_verify.json"
verification = json.loads(verify_path.read_text(encoding="utf-8"))
if (
int(verification.get("verified_windows", 0)) <= 0
or int(verification.get("mismatched_windows", -1)) != 0
or float(verification.get("max_abs_error", -1.0)) != 0.0
):
raise RuntimeError(f"Invalid exact-online hidden verification: {verify_path}")
payload["online_hidden_verification"] = verification
tmp = path.with_name(f".{path.name}.{os.getpid()}.tmp")
tmp.write_text(json.dumps(payload, ensure_ascii=False, indent=2), encoding="utf-8")
os.replace(tmp, path)
PY
cleanup_video_resume_files
rm -rf -- "$INDEX_TMP_DIR"
if [[ "$CLEAN_STAGING_AFTER_SUCCESS" == "1" ]]; then
"$PYTHON_BIN" - "$WORK_ROOT" "$OUTPUT_ROOT" <<'PY'
import json, shutil, sys
from pathlib import Path
work = Path(sys.argv[1]).resolve()
output = Path(sys.argv[2]).resolve()
marker = work / ".psi0_preprocess_work.json"
if work == output or not marker.is_file():
raise RuntimeError(f"Refusing to remove unowned staging directory: {work}")
payload = json.loads(marker.read_text(encoding="utf-8"))
if payload.get("format") != "psi0_preprocess_work_v1" or Path(payload.get("output_root", "")).resolve() != output:
raise RuntimeError(f"Invalid staging ownership marker: {marker}")
shutil.rmtree(work)
print(f"[cleanup] removed staging: {work}")
PY
fi
echo "[完成] 输出已生成并验证: $OUTPUT_ROOT"
set -Eeuo pipefail
# Archived data builder; keep relative dependencies rooted at the HFD project.
PROJECT_ROOT=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)
cd "$PROJECT_ROOT"
# =============================================================================
# 只需要改这一段
# =============================================================================
# 默认只生成 HFD audio-command 需要的 hidden:一个完整 WAV/source
# 对应一个紧凑 safetensors,内部仍是 0.6 秒窗口、0.4 秒滑动。
# 不生成逐窗口文件,不做大分片 staging/merge,也不构造无关任务。
# 旧的多任务 OSS 大分片流程可显式设置 HIDDEN_LAYOUT=oss 后继续复用。
OUTPUT_KIND="hidden" # embeds | hidden
OUTPUT_ROOT="/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/videogen/hfd_audio_command_source"
HIDDEN_LAYOUT="source" # source | oss
# 统一音频入口是 WAV。Qwen3-TTS 直接读取原始 seed_audio/*.wav;dance/2264
# 原始数据只有 MP4 音轨,因此先把历史 manifest 的规范采样(NPY/少量 MP4 fallback)
# 无损封装成 float32 WAV。
# 这样既不改变旧 hidden 的数值,又能让后续音频切 clip 只面对一种输入格式。
AUDIO_WAV_ROOT="/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/videogen/psi0_audio_wav_v26"
# 可选任务:text,audio_text,audio_only,audio_command_text,audio_command_only,
# video_clip_text,video_clip_only
SELECTED_TASKS="audio_command_text"
# 可选数据源:
# seed_text
# psi0_dance_audio
# 2264_with_audio
# qwen3_tts_seed_audio_command
# dance_1723_video_10s_clip50
# motion_example_0506_demo_clip50
# 2264_with_audio_clip50
# distilled_video_0226_clip50(目前仅建议用于 embeds;正式训练配置尚未引用它)
SELECTED_SOURCES="qwen3_tts_seed_audio_command"
CUDA_VISIBLE_DEVICES="${CUDA_VISIBLE_DEVICES:-0,1,2,3,4,5,6,7}" # embeds 模式不用 GPU
WORKERS_PER_GPU=8
TORCHRUN_LOCAL_ADDR="127.0.0.1"
BUILD_SOURCE_SPLIT=0 # HFD source hidden 不需要旧多任务 split 包
CLEAN_STAGING_AFTER_SUCCESS=1 # hidden 成功后删除中间大 shard 和 video embeds
OVERWRITE=0
# 下面通常不用改。正式数据必须保持 LIMIT=0;非零只用于逐阶段调试,
# 其输出不应直接交给训练。
LIMIT=0
VIDEO_METADATA_SHARDS=4
VIDEO_BUILD_WORKERS=64
VIDEO_EMBED_WORKERS=32
VIDEO_SAMPLE_FPS=8
VIDEO_SIZE_PIXELS=50176
VIDEO_DECODER="ffmpeg"
AUDIO_WAV_WORKERS=32
AUDIO_SAMPLE_STEP_FRAMES=1
AUDIO_WINDOW_SECONDS=0.6
# audio_command 默认不再输入整段 WAV:只保留完整的 0.6 秒窗口,
# 每 20 个 50 FPS 动作帧(0.4 秒)向后滑动一次。
AUDIO_COMMAND_FULL_AUDIO=0 # 0:切片 | 1:旧版整段语音
AUDIO_COMMAND_SAMPLE_STEP_FRAMES=20
AUDIO_COMMAND_WINDOW_SECONDS=0.6
HIDDEN_BATCH_SIZE=1
HIDDEN_SHARD_SIZE=4096
# metadata:严格复现旧 cache_omni_hidden_states_v23.sh 的样本范围。
# motion_csv:按动作 CSV 的真实帧数生成,属于新的数据定义。
AUDIO_FRAME_COUNT_SOURCE="metadata" # metadata | motion_csv
FINAL_SAMPLES_PER_SHARD=4096
FINAL_MAX_SHARD_GIB=4.5
SHUFFLE_SEED=20260710
MERGE_WORKERS=32
MERGE_SOURCE_READER_CACHE=32
VERIFY_SAMPLES=64
ONLINE_VERIFY_SOURCES=64 # 前若干完整 WAV 逐窗口复算,要求与在线 B=1 逐元素相等
VERIFY_CANONICAL_COUNTS=1 # 默认全量四任务时核对当前数据定义的精确条数
MANAGE_LOCAL_GPU_MONITOR=1
GPU_MONITOR_SCRIPT="/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/monitor/cmd.sh"
# 自动化/小测试可以用 PSI_PREP_* 临时覆盖;正常使用无需传任何参数。
OUTPUT_KIND=${PSI_PREP_OUTPUT_KIND:-$OUTPUT_KIND}
OUTPUT_ROOT=${PSI_PREP_OUTPUT_ROOT:-$OUTPUT_ROOT}
HIDDEN_LAYOUT=${PSI_PREP_HIDDEN_LAYOUT:-$HIDDEN_LAYOUT}
AUDIO_WAV_ROOT=${PSI_PREP_AUDIO_WAV_ROOT:-$AUDIO_WAV_ROOT}
SELECTED_TASKS=${PSI_PREP_TASKS:-$SELECTED_TASKS}
SELECTED_SOURCES=${PSI_PREP_SOURCES:-$SELECTED_SOURCES}
CUDA_VISIBLE_DEVICES=${PSI_PREP_CUDA_VISIBLE_DEVICES:-$CUDA_VISIBLE_DEVICES}
WORKERS_PER_GPU=${PSI_PREP_WORKERS_PER_GPU:-$WORKERS_PER_GPU}
TORCHRUN_LOCAL_ADDR=${PSI_PREP_TORCHRUN_LOCAL_ADDR:-$TORCHRUN_LOCAL_ADDR}
BUILD_SOURCE_SPLIT=${PSI_PREP_BUILD_SOURCE_SPLIT:-$BUILD_SOURCE_SPLIT}
CLEAN_STAGING_AFTER_SUCCESS=${PSI_PREP_CLEAN_STAGING:-$CLEAN_STAGING_AFTER_SUCCESS}
OVERWRITE=${PSI_PREP_OVERWRITE:-$OVERWRITE}
LIMIT=${PSI_PREP_LIMIT:-$LIMIT}
VIDEO_METADATA_SHARDS=${PSI_PREP_VIDEO_METADATA_SHARDS:-$VIDEO_METADATA_SHARDS}
VIDEO_BUILD_WORKERS=${PSI_PREP_VIDEO_BUILD_WORKERS:-$VIDEO_BUILD_WORKERS}
VIDEO_EMBED_WORKERS=${PSI_PREP_VIDEO_EMBED_WORKERS:-$VIDEO_EMBED_WORKERS}
AUDIO_WAV_WORKERS=${PSI_PREP_AUDIO_WAV_WORKERS:-$AUDIO_WAV_WORKERS}
AUDIO_SAMPLE_STEP_FRAMES=${PSI_PREP_AUDIO_SAMPLE_STEP_FRAMES:-$AUDIO_SAMPLE_STEP_FRAMES}
AUDIO_WINDOW_SECONDS=${PSI_PREP_AUDIO_WINDOW_SECONDS:-$AUDIO_WINDOW_SECONDS}
AUDIO_COMMAND_FULL_AUDIO=${PSI_PREP_AUDIO_COMMAND_FULL_AUDIO:-$AUDIO_COMMAND_FULL_AUDIO}