code-HFD / backup /prepare_omni_training_data.sh
Tinghong-Ye22's picture
Add files using upload-large-folder tool
5d9f970 verified
Raw History Blame Contribute Delete
50.8 kB
#!/usr/bin/env bash set -Eeuo pipefail # Archived data builder; keep relative dependencies rooted at the HFD project. PROJECT_ROOT=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd) cd "$PROJECT_ROOT" # ============================================================================= # 只需要改这一段 # ============================================================================= # 默认只生成 HFD audio-command 需要的 hidden:一个完整 WAV/source # 对应一个紧凑 safetensors,内部仍是 0.6 秒窗口、0.4 秒滑动。 # 不生成逐窗口文件,不做大分片 staging/merge,也不构造无关任务。 # 旧的多任务 OSS 大分片流程可显式设置 HIDDEN_LAYOUT=oss 后继续复用。 OUTPUT_KIND="hidden" # embeds | hidden OUTPUT_ROOT="/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/videogen/hfd_audio_command_source" HIDDEN_LAYOUT="source" # source | oss # 统一音频入口是 WAV。Qwen3-TTS 直接读取原始 seed_audio/*.wav;dance/2264 # 原始数据只有 MP4 音轨,因此先把历史 manifest 的规范采样(NPY/少量 MP4 fallback) # 无损封装成 float32 WAV。 # 这样既不改变旧 hidden 的数值,又能让后续音频切 clip 只面对一种输入格式。 AUDIO_WAV_ROOT="/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/videogen/psi0_audio_wav_v26" # 可选任务:text,audio_text,audio_only,audio_command_text,audio_command_only, # video_clip_text,video_clip_only SELECTED_TASKS="audio_command_text" # 可选数据源: # seed_text # psi0_dance_audio # 2264_with_audio # qwen3_tts_seed_audio_command # dance_1723_video_10s_clip50 # motion_example_0506_demo_clip50 # 2264_with_audio_clip50 # distilled_video_0226_clip50(目前仅建议用于 embeds;正式训练配置尚未引用它) SELECTED_SOURCES="qwen3_tts_seed_audio_command" CUDA_VISIBLE_DEVICES="${CUDA_VISIBLE_DEVICES:-0,1,2,3,4,5,6,7}" # embeds 模式不用 GPU WORKERS_PER_GPU=8 TORCHRUN_LOCAL_ADDR="127.0.0.1" BUILD_SOURCE_SPLIT=0 # HFD source hidden 不需要旧多任务 split 包 CLEAN_STAGING_AFTER_SUCCESS=1 # hidden 成功后删除中间大 shard 和 video embeds OVERWRITE=0 # 下面通常不用改。正式数据必须保持 LIMIT=0;非零只用于逐阶段调试, # 其输出不应直接交给训练。 LIMIT=0 VIDEO_METADATA_SHARDS=4 VIDEO_BUILD_WORKERS=64 VIDEO_EMBED_WORKERS=32 VIDEO_SAMPLE_FPS=8 VIDEO_SIZE_PIXELS=50176 VIDEO_DECODER="ffmpeg" AUDIO_WAV_WORKERS=32 AUDIO_SAMPLE_STEP_FRAMES=1 AUDIO_WINDOW_SECONDS=0.6 # audio_command 默认不再输入整段 WAV:只保留完整的 0.6 秒窗口, # 每 20 个 50 FPS 动作帧(0.4 秒)向后滑动一次。 AUDIO_COMMAND_FULL_AUDIO=0 # 0:切片 | 1:旧版整段语音 AUDIO_COMMAND_SAMPLE_STEP_FRAMES=20 AUDIO_COMMAND_WINDOW_SECONDS=0.6 HIDDEN_BATCH_SIZE=1 HIDDEN_SHARD_SIZE=4096 # metadata:严格复现旧 cache_omni_hidden_states_v23.sh 的样本范围。 # motion_csv:按动作 CSV 的真实帧数生成,属于新的数据定义。 AUDIO_FRAME_COUNT_SOURCE="metadata" # metadata | motion_csv FINAL_SAMPLES_PER_SHARD=4096 FINAL_MAX_SHARD_GIB=4.5 SHUFFLE_SEED=20260710 MERGE_WORKERS=32 MERGE_SOURCE_READER_CACHE=32 VERIFY_SAMPLES=64 ONLINE_VERIFY_SOURCES=64 # 前若干完整 WAV 逐窗口复算,要求与在线 B=1 逐元素相等 VERIFY_CANONICAL_COUNTS=1 # 默认全量四任务时核对当前数据定义的精确条数 MANAGE_LOCAL_GPU_MONITOR=1 GPU_MONITOR_SCRIPT="/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/monitor/cmd.sh" # 自动化/小测试可以用 PSI_PREP_* 临时覆盖;正常使用无需传任何参数。 OUTPUT_KIND=${PSI_PREP_OUTPUT_KIND:-$OUTPUT_KIND} OUTPUT_ROOT=${PSI_PREP_OUTPUT_ROOT:-$OUTPUT_ROOT} HIDDEN_LAYOUT=${PSI_PREP_HIDDEN_LAYOUT:-$HIDDEN_LAYOUT} AUDIO_WAV_ROOT=${PSI_PREP_AUDIO_WAV_ROOT:-$AUDIO_WAV_ROOT} SELECTED_TASKS=${PSI_PREP_TASKS:-$SELECTED_TASKS} SELECTED_SOURCES=${PSI_PREP_SOURCES:-$SELECTED_SOURCES} CUDA_VISIBLE_DEVICES=${PSI_PREP_CUDA_VISIBLE_DEVICES:-$CUDA_VISIBLE_DEVICES} WORKERS_PER_GPU=${PSI_PREP_WORKERS_PER_GPU:-$WORKERS_PER_GPU} TORCHRUN_LOCAL_ADDR=${PSI_PREP_TORCHRUN_LOCAL_ADDR:-$TORCHRUN_LOCAL_ADDR} BUILD_SOURCE_SPLIT=${PSI_PREP_BUILD_SOURCE_SPLIT:-$BUILD_SOURCE_SPLIT} CLEAN_STAGING_AFTER_SUCCESS=${PSI_PREP_CLEAN_STAGING:-$CLEAN_STAGING_AFTER_SUCCESS} OVERWRITE=${PSI_PREP_OVERWRITE:-$OVERWRITE} LIMIT=${PSI_PREP_LIMIT:-$LIMIT} VIDEO_METADATA_SHARDS=${PSI_PREP_VIDEO_METADATA_SHARDS:-$VIDEO_METADATA_SHARDS} VIDEO_BUILD_WORKERS=${PSI_PREP_VIDEO_BUILD_WORKERS:-$VIDEO_BUILD_WORKERS} VIDEO_EMBED_WORKERS=${PSI_PREP_VIDEO_EMBED_WORKERS:-$VIDEO_EMBED_WORKERS} AUDIO_WAV_WORKERS=${PSI_PREP_AUDIO_WAV_WORKERS:-$AUDIO_WAV_WORKERS} AUDIO_SAMPLE_STEP_FRAMES=${PSI_PREP_AUDIO_SAMPLE_STEP_FRAMES:-$AUDIO_SAMPLE_STEP_FRAMES} AUDIO_WINDOW_SECONDS=${PSI_PREP_AUDIO_WINDOW_SECONDS:-$AUDIO_WINDOW_SECONDS} AUDIO_COMMAND_FULL_AUDIO=${PSI_PREP_AUDIO_COMMAND_FULL_AUDIO:-$AUDIO_COMMAND_FULL_AUDIO} AUDIO_COMMAND_SAMPLE_STEP_FRAMES=${PSI_PREP_AUDIO_COMMAND_SAMPLE_STEP_FRAMES:-$AUDIO_COMMAND_SAMPLE_STEP_FRAMES} AUDIO_COMMAND_WINDOW_SECONDS=${PSI_PREP_AUDIO_COMMAND_WINDOW_SECONDS:-$AUDIO_COMMAND_WINDOW_SECONDS} HIDDEN_BATCH_SIZE=${PSI_PREP_HIDDEN_BATCH_SIZE:-$HIDDEN_BATCH_SIZE} HIDDEN_SHARD_SIZE=${PSI_PREP_HIDDEN_SHARD_SIZE:-$HIDDEN_SHARD_SIZE} AUDIO_FRAME_COUNT_SOURCE=${PSI_PREP_AUDIO_FRAME_COUNT_SOURCE:-$AUDIO_FRAME_COUNT_SOURCE} FINAL_SAMPLES_PER_SHARD=${PSI_PREP_FINAL_SAMPLES_PER_SHARD:-$FINAL_SAMPLES_PER_SHARD} FINAL_MAX_SHARD_GIB=${PSI_PREP_FINAL_MAX_SHARD_GIB:-$FINAL_MAX_SHARD_GIB} MERGE_WORKERS=${PSI_PREP_MERGE_WORKERS:-$MERGE_WORKERS} VERIFY_SAMPLES=${PSI_PREP_VERIFY_SAMPLES:-$VERIFY_SAMPLES} ONLINE_VERIFY_SOURCES=${PSI_PREP_ONLINE_VERIFY_SOURCES:-$ONLINE_VERIFY_SOURCES} VERIFY_CANONICAL_COUNTS=${PSI_PREP_VERIFY_CANONICAL_COUNTS:-$VERIFY_CANONICAL_COUNTS} MANAGE_LOCAL_GPU_MONITOR=${PSI_PREP_MANAGE_GPU_MONITOR:-$MANAGE_LOCAL_GPU_MONITOR} DRY_RUN=${PSI_PREP_DRY_RUN:-0} # ============================================================================= # 固定的数据位置和模型配置 # ============================================================================= PYTHON_BIN=${PSI_PREP_PYTHON_BIN:-$PROJECT_ROOT/.venv-psi/bin/python} MODEL_PATH=${PSI_PREP_MODEL_PATH:-/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/team_workspace/public_ckpt/Qwen/Qwen2.5-Omni-3B} SEED_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/Psi0/dataset/seed_all SEED_METADATA=$SEED_ROOT/metadata/seed_metadata_v004.csv DANCE_MOTION_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/GR00T-WholeBodyControl/gear_sonic_deploy/reference/converted/psi0_dance/gvhmr DANCE_AV_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/dance_1723_video_psi0/dance_1723_video_10s MOTION_EXAMPLE_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/videogen/motion_example_0506_demo AUDIO_2264_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/videogen/2264_with_audio QWEN3_TTS_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/qwen3-tts DISTILLED_VIDEO_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/videogen/distilled_video_0226 if [[ $# -ne 0 ]]; then echo "本脚本不接收位置参数;请直接修改顶部配置。" >&2 exit 2 fi normalize_csv() { local value=${1//[[:space:]]/} value=${value#,} value=${value%,} printf '%s' "$value" } csv_has() { local values=",$(normalize_csv "$1")," [[ "$values" == *",$2,"* ]] } validate_csv_members() { local label=$1 local values=$2 shift 2 local allowed=",$*," allowed=${allowed// /,} local token IFS=',' read -r -a tokens <<< "$(normalize_csv "$values")" for token in "${tokens[@]}"; do [[ -n "$token" ]] || continue if [[ "$allowed" != *",$token,"* ]]; then echo "未知${label}: $token" >&2 exit 2 fi done } SELECTED_TASKS=$(normalize_csv "$SELECTED_TASKS") SELECTED_SOURCES=$(normalize_csv "$SELECTED_SOURCES") validate_csv_members "任务" "$SELECTED_TASKS" \ text audio_text audio_only audio_command_text audio_command_only video_clip_text video_clip_only validate_csv_members "数据源" "$SELECTED_SOURCES" \ seed_text psi0_dance_audio 2264_with_audio qwen3_tts_seed_audio_command \ dance_1723_video_10s_clip50 motion_example_0506_demo_clip50 \ 2264_with_audio_clip50 distilled_video_0226_clip50 if [[ "$OUTPUT_KIND" != "embeds" && "$OUTPUT_KIND" != "hidden" ]]; then echo "OUTPUT_KIND 只能是 embeds 或 hidden" >&2 exit 2 fi if [[ "$HIDDEN_LAYOUT" != "source" && "$HIDDEN_LAYOUT" != "oss" ]]; then echo "HIDDEN_LAYOUT 只能是 source 或 oss" >&2 exit 2 fi if [[ "$AUDIO_FRAME_COUNT_SOURCE" != "metadata" && "$AUDIO_FRAME_COUNT_SOURCE" != "motion_csv" ]]; then echo "AUDIO_FRAME_COUNT_SOURCE 只能是 metadata 或 motion_csv" >&2 exit 2 fi if (( AUDIO_WAV_WORKERS <= 0 || AUDIO_SAMPLE_STEP_FRAMES <= 0 || AUDIO_COMMAND_SAMPLE_STEP_FRAMES <= 0 )); then echo "AUDIO_WAV_WORKERS、AUDIO_SAMPLE_STEP_FRAMES 和 AUDIO_COMMAND_SAMPLE_STEP_FRAMES 必须为正整数" >&2 exit 2 fi if (( WORKERS_PER_GPU <= 0 || ONLINE_VERIFY_SOURCES <= 0 )); then echo "WORKERS_PER_GPU 和 ONLINE_VERIFY_SOURCES 必须为正整数" >&2 exit 2 fi if [[ "$AUDIO_COMMAND_FULL_AUDIO" != "0" && "$AUDIO_COMMAND_FULL_AUDIO" != "1" ]]; then echo "AUDIO_COMMAND_FULL_AUDIO 只能是 0 或 1" >&2 exit 2 fi if [[ -z "$SELECTED_TASKS" || -z "$SELECTED_SOURCES" ]]; then echo "SELECTED_TASKS 和 SELECTED_SOURCES 不能为空" >&2 exit 2 fi if [[ ! -x "$PYTHON_BIN" ]]; then echo "Python 不可执行: $PYTHON_BIN" >&2 exit 2 fi if ! "$PYTHON_BIN" - "$AUDIO_WINDOW_SECONDS" "$AUDIO_COMMAND_WINDOW_SECONDS" <<'PY' import sys raise SystemExit(0 if float(sys.argv[1]) > 0 and float(sys.argv[2]) > 0 else 1) PY then echo "AUDIO_WINDOW_SECONDS 和 AUDIO_COMMAND_WINDOW_SECONDS 必须为正数" >&2 exit 2 fi if csv_has "$SELECTED_TASKS" text && ! csv_has "$SELECTED_SOURCES" seed_text; then echo "选择了 text 任务,但没有选择 seed_text 数据源" >&2 exit 2 fi if { csv_has "$SELECTED_TASKS" audio_text || csv_has "$SELECTED_TASKS" audio_only; } \ && ! csv_has "$SELECTED_SOURCES" psi0_dance_audio \ && ! csv_has "$SELECTED_SOURCES" 2264_with_audio; then echo "选择了 audio 任务,但没有选择 audio 数据源" >&2 exit 2 fi if { csv_has "$SELECTED_TASKS" audio_command_text || csv_has "$SELECTED_TASKS" audio_command_only; } \ && ! csv_has "$SELECTED_SOURCES" qwen3_tts_seed_audio_command; then echo "选择了 audio_command 任务,但没有选择 qwen3_tts_seed_audio_command" >&2 exit 2 fi if [[ "$OUTPUT_KIND" == "hidden" ]] && csv_has "$SELECTED_SOURCES" distilled_video_0226_clip50; then echo "distilled_video_0226_clip50 尚未接入正式训练配置,hidden 模式拒绝静默生成无消费者的数据。" >&2 exit 2 fi if [[ "$HIDDEN_LAYOUT" == "source" ]]; then if [[ "$OUTPUT_KIND" != "hidden" \ || "$SELECTED_TASKS" != "audio_command_text" \ || "$SELECTED_SOURCES" != "qwen3_tts_seed_audio_command" ]]; then echo "source 布局只用于 HFD 的 qwen3_tts_seed_audio_command/audio_command_text。" >&2 echo "其他多任务组合请显式设置 PSI_PREP_HIDDEN_LAYOUT=oss。" >&2 exit 2 fi if [[ "$AUDIO_COMMAND_FULL_AUDIO" != "0" ]]; then echo "source 布局必须按流式窗口导出,AUDIO_COMMAND_FULL_AUDIO 必须为 0。" >&2 exit 2 fi if [[ "$HIDDEN_BATCH_SIZE" != "1" ]]; then echo "HFD source hidden 必须逐窗口执行真实在线 B=1 编码,HIDDEN_BATCH_SIZE 必须为 1。" >&2 exit 2 fi if [[ "$LIMIT" != "0" \ && "$OUTPUT_ROOT" == "/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/videogen/hfd_audio_command_source" ]]; then echo "有限 source 测试不能写入正式目录;请同时设置 PSI_PREP_OUTPUT_ROOT 到临时目录。" >&2 exit 2 fi if ! "$PYTHON_BIN" - "$AUDIO_COMMAND_WINDOW_SECONDS" \ "$AUDIO_COMMAND_SAMPLE_STEP_FRAMES" <<'PY' import math, sys valid = ( math.isclose(float(sys.argv[1]), 0.6, rel_tol=0.0, abs_tol=1e-9) and int(sys.argv[2]) == 20 ) raise SystemExit(0 if valid else 1) PY then echo "HFD source 布局固定使用 0.6 秒窗口和 20 帧/0.4 秒步长。" >&2 exit 2 fi fi OUTPUT_ROOT=$($PYTHON_BIN -c 'import os,sys; print(os.path.abspath(os.path.expanduser(sys.argv[1])))' "$OUTPUT_ROOT") AUDIO_WAV_ROOT=$($PYTHON_BIN -c 'import os,sys; print(os.path.abspath(os.path.expanduser(sys.argv[1])))' "$AUDIO_WAV_ROOT") WORK_ROOT="${OUTPUT_ROOT}.work" if [[ "$OUTPUT_KIND" == "embeds" ]]; then VIDEO_CACHE_ROOT="$OUTPUT_ROOT" FINAL_VIDEO_SUPPORT_ROOT="$OUTPUT_ROOT" else VIDEO_CACHE_ROOT="$WORK_ROOT/video_clip_dense_stride1" FINAL_VIDEO_SUPPORT_ROOT="$OUTPUT_ROOT/training_support/video_clip_dense_stride1" fi HIDDEN_STAGING_ROOT="$WORK_ROOT/hidden_source_shards" if [[ "$HIDDEN_LAYOUT" == "source" ]]; then HIDDEN_EXPORT_ROOT="$OUTPUT_ROOT" else HIDDEN_EXPORT_ROOT="$HIDDEN_STAGING_ROOT" fi SOURCE_CONFIG_FILE="$WORK_ROOT/source_presets.json" SOURCE_SPLIT_ROOT="$OUTPUT_ROOT/training_support/source_split_v1" INDEX_TMP_DIR="/tmp/psi0_unified_preprocess_indexes.$$" DANCE_AUDIO_WAV_ROOT="$AUDIO_WAV_ROOT/psi0_dance_audio" AUDIO_2264_WAV_ROOT="$AUDIO_WAV_ROOT/2264_with_audio" video_task_selected=0 if csv_has "$SELECTED_TASKS" video_clip_text || csv_has "$SELECTED_TASKS" video_clip_only; then video_task_selected=1 fi VIDEO_SOURCES=() for source in dance_1723_video_10s_clip50 motion_example_0506_demo_clip50 2264_with_audio_clip50 distilled_video_0226_clip50; do if csv_has "$SELECTED_SOURCES" "$source"; then VIDEO_SOURCES+=("$source") fi done if [[ "$OUTPUT_KIND" == "embeds" && ${#VIDEO_SOURCES[@]} -eq 0 ]]; then echo "embeds 模式至少要选择一个 video_clip 数据源" >&2 exit 2 fi if [[ $video_task_selected -eq 1 && ${#VIDEO_SOURCES[@]} -eq 0 ]]; then echo "选择了 video_clip 任务,但没有选择 video_clip 数据源" >&2 exit 2 fi echo "================================================================================" echo "Psi0 统一离线预处理" echo "================================================================================" echo "输出类型: $OUTPUT_KIND" echo "输出目录: $OUTPUT_ROOT" echo "hidden 布局: $HIDDEN_LAYOUT" echo "WAV 输入目录: $AUDIO_WAV_ROOT" echo "任务: $SELECTED_TASKS" echo "数据源: $SELECTED_SOURCES" echo "video 工作目录: $VIDEO_CACHE_ROOT" echo "hidden 写入: $HIDDEN_EXPORT_ROOT" echo "torchrun 地址: $TORCHRUN_LOCAL_ADDR" echo "audio 帧范围: $AUDIO_FRAME_COUNT_SOURCE" echo "audio 窗口: ${AUDIO_WINDOW_SECONDS}s,stride=${AUDIO_SAMPLE_STEP_FRAMES} motion frame(s)" echo "audio-command: full_audio=$AUDIO_COMMAND_FULL_AUDIO,${AUDIO_COMMAND_WINDOW_SECONDS}s,stride=${AUDIO_COMMAND_SAMPLE_STEP_FRAMES} motion frame(s)" echo "Qwen 编码契约: batch=$HIDDEN_BATCH_SIZE,bf16,thinker 最后一层 hidden" echo "在线精确复核: ${ONLINE_VERIFY_SOURCES} 个完整 WAV,要求逐元素完全一致" echo "source split: $BUILD_SOURCE_SPLIT" echo "测试条数限制: $LIMIT" echo "================================================================================" if [[ "$DRY_RUN" == "1" ]]; then echo "[dry-run] 配置检查通过,不写文件。" exit 0 fi export PYTHONPATH="$PROJECT_ROOT/src${PYTHONPATH:+:$PYTHONPATH}" export PYTHONUNBUFFERED=1 export TOKENIZERS_PARALLELISM=false export OMP_NUM_THREADS=4 export MKL_NUM_THREADS=1 export OPENBLAS_NUM_THREADS=1 export NUMEXPR_NUM_THREADS=1 ulimit -n 1048576 2>/dev/null || true mkdir -p "$OUTPUT_ROOT" "$WORK_ROOT" "$INDEX_TMP_DIR" $PYTHON_BIN - "$WORK_ROOT/.psi0_preprocess_work.json" "$OUTPUT_ROOT" <<'PY' import json, os, sys from pathlib import Path path = Path(sys.argv[1]) payload = {"format": "psi0_preprocess_work_v1", "output_root": str(Path(sys.argv[2]).resolve())} tmp = path.with_name(f".{path.name}.{os.getpid()}.tmp") tmp.write_text(json.dumps(payload, indent=2), encoding="utf-8") os.replace(tmp, path) PY prepare_audio_wav_source() { local source=$1 local dataset_root metadata_jsonl output_root case "$source" in psi0_dance_audio) dataset_root="$DANCE_AV_ROOT" metadata_jsonl="$DANCE_AV_ROOT/metadata_audio_npy.jsonl" output_root="$DANCE_AUDIO_WAV_ROOT" ;; 2264_with_audio) dataset_root="$AUDIO_2264_ROOT" metadata_jsonl="$AUDIO_2264_ROOT/metadata_audio_npy.jsonl" output_root="$AUDIO_2264_WAV_ROOT" ;; *) echo "未知 WAV audio source: $source" >&2 return 2 ;; esac local -a args=( scripts/data/materialize_audio_wav.py --dataset-root "$dataset_root" --metadata-jsonl "$metadata_jsonl" --output-root "$output_root" --output-metadata-jsonl metadata.jsonl --input-kind auto --input-path-key audio_path --output-path-key audio_path --absolute-path-keys converted_path,motion_dir,motion_path --strip-input-prefix audio_npy_yth --output-audio-dir audio --sample-rate 16000 --num-workers "$AUDIO_WAV_WORKERS" --skip-media-without-audio ) if [[ "$LIMIT" != "0" ]]; then args+=(--limit "$LIMIT") fi if [[ "$OVERWRITE" == "1" ]]; then args+=(--overwrite) fi echo "[audio wav] source=$source output=$output_root" "$PYTHON_BIN" -u "${args[@]}" } if csv_has "$SELECTED_TASKS" audio_text || csv_has "$SELECTED_TASKS" audio_only; then csv_has "$SELECTED_SOURCES" psi0_dance_audio && prepare_audio_wav_source psi0_dance_audio csv_has "$SELECTED_SOURCES" 2264_with_audio && prepare_audio_wav_source 2264_with_audio fi if csv_has "$SELECTED_TASKS" audio_command_text || csv_has "$SELECTED_TASKS" audio_command_only; then [[ -s "$QWEN3_TTS_ROOT/metadata.jsonl" && -d "$QWEN3_TTS_ROOT/seed_audio" ]] || { echo "Qwen3-TTS 原始 WAV 或 metadata.jsonl 不存在: $QWEN3_TTS_ROOT" >&2 exit 1 } fi common_video_args() { VIDEO_ARGS=( --clip-frames 50 --action-chunk-frames 30 --chunk-stride-frames 1 --motion-fps 50 --clips-per-video 1 --clip-selection-mode all --random-seed 42 --middle-fraction 0.8 --num-workers "$VIDEO_BUILD_WORKERS" --chunk-workers 1 --metadata-only --shard-num "$VIDEO_METADATA_SHARDS" ) if [[ "$LIMIT" != "0" ]]; then VIDEO_ARGS+=(--limit "$LIMIT") fi if [[ "$OVERWRITE" == "1" ]]; then VIDEO_ARGS+=(--overwrite) fi } video_output_dir() { case "$1" in dance_1723_video_10s_clip50) printf '%s/dance_1723_video_10s' "$VIDEO_CACHE_ROOT" ;; motion_example_0506_demo_clip50) printf '%s/motion_example_0506_demo' "$VIDEO_CACHE_ROOT" ;; 2264_with_audio_clip50) printf '%s/2264_with_audio' "$VIDEO_CACHE_ROOT" ;; distilled_video_0226_clip50) printf '%s/distilled_video_0226' "$VIDEO_CACHE_ROOT" ;; *) echo "未知 video source: $1" >&2; return 2 ;; esac } build_video_metadata() { local source=$1 local output_dir output_dir=$(video_output_dir "$source") local shard_id for ((shard_id=0; shard_id<VIDEO_METADATA_SHARDS; shard_id++)); do common_video_args VIDEO_ARGS+=(--output-root "$output_dir" --shard-id "$shard_id") case "$source" in dance_1723_video_10s_clip50) VIDEO_ARGS+=( --dataset dance --dance-motion-root "$DANCE_MOTION_ROOT" --dance-video-root "$DANCE_AV_ROOT" ) ;; motion_example_0506_demo_clip50) VIDEO_ARGS+=( --dataset motion_example --motion-example-root "$MOTION_EXAMPLE_ROOT" --metadata-jsonl metadata.jsonl ) ;; 2264_with_audio_clip50) VIDEO_ARGS+=( --dataset manifest --manifest-root "$AUDIO_2264_ROOT" --metadata-jsonl metadata.jsonl --motion-metadata-jsonl metadata_convert.jsonl --motion-type-default 2264_with_audio --rel-name-style name ) ;; distilled_video_0226_clip50) VIDEO_ARGS+=( --dataset manifest --manifest-root "$DISTILLED_VIDEO_ROOT" --metadata-jsonl metadata.jsonl --motion-metadata-jsonl metadata_convert.jsonl --motion-type-default distilled_video_0226 --rel-name-style name ) ;; esac echo "[video metadata] source=$source shard=$shard_id/$VIDEO_METADATA_SHARDS" "$PYTHON_BIN" scripts/data/build_video_clip_dataset.py "${VIDEO_ARGS[@]}" done [[ -s "$output_dir/metadata.jsonl" ]] || { echo "video metadata 未生成: $output_dir/metadata.jsonl" >&2 return 1 } } cache_video_embeds() { local source=$1 local output_dir output_dir=$(video_output_dir "$source") local shard_id metadata_name for ((shard_id=0; shard_id<VIDEO_METADATA_SHARDS; shard_id++)); do if [[ "$VIDEO_METADATA_SHARDS" == "1" ]]; then metadata_name=metadata.jsonl else printf -v metadata_name 'metadata.shard-%05d-of-%05d.jsonl' "$shard_id" "$VIDEO_METADATA_SHARDS" fi EMBED_ARGS=( --dataset-root "$output_dir" --metadata-jsonl "$metadata_name" --embed-root "$output_dir/embeds" --model-path "$MODEL_PATH" --video-sample-fps "$VIDEO_SAMPLE_FPS" --video-size-pixels "$VIDEO_SIZE_PIXELS" --video-decoder "$VIDEO_DECODER" --dtype float16 --num-workers "$VIDEO_EMBED_WORKERS" --existing-cache-check size --shard-num "$VIDEO_METADATA_SHARDS" --shard-id "$shard_id" --input-already-sharded --direct-source-clips ) if [[ "$LIMIT" != "0" ]]; then EMBED_ARGS+=(--limit "$LIMIT") fi if [[ "$OVERWRITE" == "1" ]]; then EMBED_ARGS+=(--overwrite) fi echo "[video embeds] source=$source shard=$shard_id/$VIDEO_METADATA_SHARDS" "$PYTHON_BIN" "$PROJECT_ROOT/backup/cache_video_omni_embeds.py" "${EMBED_ARGS[@]}" done [[ -s "$output_dir/embeds/manifest.json" ]] || { echo "video embed manifest 未生成: $output_dir/embeds/manifest.json" >&2 return 1 } } if [[ $video_task_selected -eq 1 || "$OUTPUT_KIND" == "embeds" ]]; then for source in "${VIDEO_SOURCES[@]}"; do build_video_metadata "$source" cache_video_embeds "$source" done fi write_source_config() { export PSI_PREP_SELECTED_SOURCES="$SELECTED_SOURCES" export PSI_PREP_VIDEO_CACHE_ROOT="$VIDEO_CACHE_ROOT" export PSI_PREP_SEED_ROOT="$SEED_ROOT" export PSI_PREP_SEED_METADATA="$SEED_METADATA" export PSI_PREP_QWEN3_TTS_ROOT="$QWEN3_TTS_ROOT" export PSI_PREP_DANCE_AUDIO_WAV_ROOT="$DANCE_AUDIO_WAV_ROOT" export PSI_PREP_2264_AUDIO_WAV_ROOT="$AUDIO_2264_WAV_ROOT" "$PYTHON_BIN" - "$SOURCE_CONFIG_FILE" <<'PY' import json, os, sys from pathlib import Path selected = {value for value in os.environ["PSI_PREP_SELECTED_SOURCES"].split(",") if value} video_root = Path(os.environ["PSI_PREP_VIDEO_CACHE_ROOT"]) seed_root = Path(os.environ["PSI_PREP_SEED_ROOT"]) qwen_root = Path(os.environ["PSI_PREP_QWEN3_TTS_ROOT"]) dance_wav_root = Path(os.environ["PSI_PREP_DANCE_AUDIO_WAV_ROOT"]) root_2264_wav = Path(os.environ["PSI_PREP_2264_AUDIO_WAV_ROOT"]) sources = [ { "name": "seed_text", "modality": "text", "dataset_root": str(seed_root), "manifest_path": os.environ["PSI_PREP_SEED_METADATA"], "allowed_tasks": ["text"], }, { "name": "2264_with_audio", "modality": "audio", "dataset_root": str(root_2264_wav), "manifest_path": str(root_2264_wav / "metadata.jsonl"), "allowed_tasks": ["audio_text", "audio_only"], "audio_file_extensions": [".wav"], }, { "name": "psi0_dance_audio", "modality": "audio", "dataset_root": str(dance_wav_root), "manifest_path": str(dance_wav_root / "metadata.jsonl"), "allowed_tasks": ["audio_text", "audio_only"], "audio_file_extensions": [".wav"], }, { "name": "qwen3_tts_seed_audio_command", "modality": "audio", "dataset_root": str(qwen_root), "manifest_path": str(qwen_root / "metadata.jsonl"), "allowed_tasks": ["audio_command_text", "audio_command_only"], "instruction_key": "content_short_description", "is_audio_dir_manifest": True, "audio_file_extensions": [".wav"], }, ] for name, directory in ( ("dance_1723_video_10s_clip50", "dance_1723_video_10s"), ("motion_example_0506_demo_clip50", "motion_example_0506_demo"), ("2264_with_audio_clip50", "2264_with_audio"), ("distilled_video_0226_clip50", "distilled_video_0226"), ): root = video_root / directory sources.append({ "name": name, "modality": "video_clip", "dataset_root": str(root), "manifest_path": str(root / "metadata.jsonl"), "allowed_tasks": ["video_clip_text", "video_clip_only"], "instruction_key": "video_caption", "video_embed_dir": str(root / "embeds"), }) sources = [source for source in sources if source["name"] in selected] path = Path(sys.argv[1]) path.parent.mkdir(parents=True, exist_ok=True) tmp = path.with_name(f".{path.name}.{os.getpid()}.tmp") tmp.write_text(json.dumps({"sources": sources}, ensure_ascii=False, indent=2), encoding="utf-8") os.replace(tmp, path) PY } monitor_stopped=0 restore_gpu_monitor() { if [[ "$monitor_stopped" == "1" && "$MANAGE_LOCAL_GPU_MONITOR" == "1" && -f "$GPU_MONITOR_SCRIPT" ]]; then bash "$GPU_MONITOR_SCRIPT" monitor_stopped=0 fi } trap restore_gpu_monitor EXIT stop_local_gpu_monitor() { if [[ "$MANAGE_LOCAL_GPU_MONITOR" == "1" ]]; then local monitor_dir=${GPU_MONITOR_SCRIPT%/*} pkill -f "$monitor_dir/[m]onitor.sh" 2>/dev/null || true pkill -f "$monitor_dir/[m]onitor.py" 2>/dev/null || true monitor_stopped=1 echo "[GPU] 已停止本项目占卡 monitor;不检查、不阻止其他 GPU 进程。" fi } if [[ "$OUTPUT_KIND" == "hidden" ]]; then write_source_config stop_local_gpu_monitor export CUDA_VISIBLE_DEVICES export NCCL_DEBUG=WARN export PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True VISIBLE_GPU_COUNT=$( "$PYTHON_BIN" - <<'PY' import torch print(torch.cuda.device_count()) PY ) if [[ ! "$VISIBLE_GPU_COUNT" =~ ^[1-9][0-9]*$ ]]; then echo "PyTorch 没有识别到可用 CUDA GPU(device_count=$VISIBLE_GPU_COUNT)。" >&2 exit 2 fi NPROC=$(( VISIBLE_GPU_COUNT * WORKERS_PER_GPU )) if [[ "$HIDDEN_LAYOUT" == "source" ]]; then HIDDEN_SAVE_WORKERS=4 HIDDEN_MAX_PENDING_SAVES=8 else HIDDEN_SAVE_WORKERS=16 HIDDEN_MAX_PENDING_SAVES=32 fi HIDDEN_ARGS=( scripts/data/cache_omni_hidden_states.py --mode export --model-path "$MODEL_PATH" --output-root "$HIDDEN_EXPORT_ROOT" --tasks "$SELECTED_TASKS" --sources "$SELECTED_SOURCES" --source-config-file "$SOURCE_CONFIG_FILE" --motion-fps 50 --audio-sample-step-frames "$AUDIO_SAMPLE_STEP_FRAMES" --audio-frame-count-source "$AUDIO_FRAME_COUNT_SOURCE" --audio-window-seconds "$AUDIO_WINDOW_SECONDS" --audio-left-context-seconds 0.0 --audio-command-sample-step-frames "$AUDIO_COMMAND_SAMPLE_STEP_FRAMES" --audio-command-window-seconds "$AUDIO_COMMAND_WINDOW_SECONDS" --audio-sampling-rate 16000 --audio-batch-size "$HIDDEN_BATCH_SIZE" --text-batch-size "$HIDDEN_BATCH_SIZE" --video-batch-size "$HIDDEN_BATCH_SIZE" --num-io-workers 8 --num-save-workers "$HIDDEN_SAVE_WORKERS" --max-pending-save-batches "$HIDDEN_MAX_PENDING_SAVES" --save-dtype bf16 --storage-format "$HIDDEN_LAYOUT" ) if [[ "$HIDDEN_LAYOUT" == "oss" ]]; then HIDDEN_ARGS+=(--shard-size "$HIDDEN_SHARD_SIZE") fi if [[ $video_task_selected -eq 1 ]]; then HIDDEN_ARGS+=(--confirm-full-video) fi if [[ "$AUDIO_COMMAND_FULL_AUDIO" == "1" ]]; then HIDDEN_ARGS+=(--audio-command-full-audio) else HIDDEN_ARGS+=(--no-audio-command-full-audio) fi if [[ "$LIMIT" != "0" ]]; then if [[ "$HIDDEN_LAYOUT" == "source" ]]; then HIDDEN_ARGS+=(--limit-sources "$LIMIT") else HIDDEN_ARGS+=(--limit "$LIMIT") fi fi if [[ "$OVERWRITE" == "1" ]]; then HIDDEN_ARGS+=(--overwrite) fi echo "[hidden export] requested_GPUs=$CUDA_VISIBLE_DEVICES detected_GPUs=$VISIBLE_GPU_COUNT workers_per_gpu=$WORKERS_PER_GPU nproc=$NPROC" "$PYTHON_BIN" -m torch.distributed.run --standalone \ --local_addr="$TORCHRUN_LOCAL_ADDR" --nproc_per_node="$NPROC" \ "${HIDDEN_ARGS[@]}" if [[ "$HIDDEN_LAYOUT" == "source" ]]; then echo "[hidden verify] 用真实在线 B=1 Qwen 路径复算 ${ONLINE_VERIFY_SOURCES} 个完整 WAV" "$PYTHON_BIN" -u scripts/data/cache_omni_hidden_states.py \ --mode verify \ --model-path "$MODEL_PATH" \ --output-root "$HIDDEN_EXPORT_ROOT" \ --tasks "$SELECTED_TASKS" \ --sources "$SELECTED_SOURCES" \ --source-config-file "$SOURCE_CONFIG_FILE" \ --motion-fps 50 \ --audio-frame-count-source "$AUDIO_FRAME_COUNT_SOURCE" \ --audio-command-sample-step-frames "$AUDIO_COMMAND_SAMPLE_STEP_FRAMES" \ --audio-command-window-seconds "$AUDIO_COMMAND_WINDOW_SECONDS" \ --audio-sampling-rate 16000 \ --audio-batch-size 1 \ --save-dtype bf16 \ --storage-format source \ --verify-sources "$ONLINE_VERIFY_SOURCES" \ --no-audio-command-full-audio fi restore_gpu_monitor if [[ "$HIDDEN_LAYOUT" == "oss" ]]; then MERGE_ARGS=( scripts/data/merge_omni_hidden_for_oss.py --src-root "$HIDDEN_STAGING_ROOT" --dst-root "$OUTPUT_ROOT" --group-by task --samples-per-shard "$FINAL_SAMPLES_PER_SHARD" --max-shard-gib "$FINAL_MAX_SHARD_GIB" --workers "$MERGE_WORKERS" --max-pending "$((MERGE_WORKERS * 2))" --executor process --start-method fork --seed "$SHUFFLE_SEED" --verify-samples "$VERIFY_SAMPLES" --source-reader-cache-size "$MERGE_SOURCE_READER_CACHE" --index-tmp-dir "$INDEX_TMP_DIR" --write-compat-indexes --no-write-jsonl-index ) IFS=',' read -r -a task_list <<< "$SELECTED_TASKS" for task in "${task_list[@]}"; do MERGE_ARGS+=(--include-task "$task") done if [[ "$OVERWRITE" == "1" ]]; then MERGE_ARGS+=(--overwrite) fi "$PYTHON_BIN" -u "${MERGE_ARGS[@]}" if [[ "$VERIFY_CANONICAL_COUNTS" == "1" && "$LIMIT" == "0" ]]; then "$PYTHON_BIN" - "$OUTPUT_ROOT" "$SELECTED_TASKS" "$SELECTED_SOURCES" "$AUDIO_FRAME_COUNT_SOURCE" \ "$AUDIO_COMMAND_FULL_AUDIO" "$AUDIO_COMMAND_SAMPLE_STEP_FRAMES" "$AUDIO_COMMAND_WINDOW_SECONDS" <<'PY' import json import sys from pathlib import Path root = Path(sys.argv[1]) tasks = {value for value in sys.argv[2].split(",") if value} sources = {value for value in sys.argv[3].split(",") if value} frame_source = sys.argv[4] command_full_audio = sys.argv[5] == "1" command_step_frames = int(sys.argv[6]) command_window_seconds = float(sys.argv[7]) canonical_tasks = {"text", "audio_text", "audio_command_text", "video_clip_text"} canonical_sources = { "seed_text", "psi0_dance_audio", "2264_with_audio", "qwen3_tts_seed_audio_command", "dance_1723_video_10s_clip50", "motion_example_0506_demo_clip50", "2264_with_audio_clip50", } is_windowed_command = ( not command_full_audio and command_step_frames == 20 and command_window_seconds == 0.6 ) if ( tasks == canonical_tasks and sources == canonical_sources and frame_source == "metadata" and is_windowed_command ): expected = { "text": 142_220, "audio_text": 14_812_682, "audio_command_text": 304_986, "video_clip_text": 18_558_588, } elif ( tasks == {"audio_command_text"} and sources == {"qwen3_tts_seed_audio_command"} and is_windowed_command ): expected = {"audio_command_text": 304_986} else: expected = {} print("[canonical counts] 非默认组合,跳过固定条数检查。") if expected: for task, expected_count in expected.items(): marker = root / "tasks" / task / ".merge_complete.json" payload = json.loads(marker.read_text(encoding="utf-8")) actual = int(payload.get("samples", -1)) if actual != expected_count: raise RuntimeError( f"Canonical sample count mismatch for {task}: {actual} != {expected_count}" ) print(f"[canonical counts] {task}: {actual:,}(符合当前默认口径)") PY fi else "$PYTHON_BIN" - "$OUTPUT_ROOT" "$NPROC" "$LIMIT" <<'PY' import sqlite3 import sys from pathlib import Path root = Path(sys.argv[1]) world_size = int(sys.argv[2]) limit_sources = int(sys.argv[3]) source_dir = ( root / "audio" / "qwen3_tts_seed_audio_command" / "audio_command_text" / "sources" ) indexes = sorted(source_dir.glob(f"index.ws{world_size}.rank*.sqlite")) if len(indexes) != world_size: raise RuntimeError( f"Source index count mismatch: {len(indexes)} != {world_size} under {source_dir}" ) rows = {} for index_path in indexes: with sqlite3.connect(f"file:{index_path}?mode=ro", uri=True) as conn: for row in conn.execute( "SELECT key, path, count, window_seconds, step_frames, " "motion_fps, sampling_rate FROM sources" ): key = str(row[0]) value = ( str(row[1]), int(row[2]), float(row[3]), int(row[4]), int(row[5]), int(row[6]), ) previous = rows.get(key) if previous is not None and previous != value: raise RuntimeError(f"Conflicting source row for {key}") rows[key] = value window_total = 0 for key, (relative_path, count, window_seconds, step_frames, fps, rate) in rows.items(): if ( count <= 0 or window_seconds != 0.6 or step_frames != 20 or fps != 50 or rate != 16000 ): raise RuntimeError( f"Invalid HFD timing/index row for {key}: " f"{count}, {window_seconds}, {step_frames}, {fps}, {rate}" ) path = source_dir / relative_path if not path.is_file(): raise FileNotFoundError(path) window_total += count if limit_sources == 0: if len(rows) != 41_694: raise RuntimeError(f"HFD source count mismatch: {len(rows):,} != 41,694") if window_total != 304_986: raise RuntimeError( f"HFD window count mismatch: {window_total:,} != 304,986" ) print( f"[source verify] sources={len(rows):,} windows={window_total:,} " f"indexes={len(indexes)} timing=0.6s/0.4s" ) PY fi fi copy_hidden_training_metadata() { [[ "$OUTPUT_KIND" == "hidden" ]] || return 0 [[ $video_task_selected -eq 1 ]] || return 0 local source src_dir dst_dir for source in "${VIDEO_SOURCES[@]}"; do src_dir=$(video_output_dir "$source") case "$source" in dance_1723_video_10s_clip50) dst_dir="$FINAL_VIDEO_SUPPORT_ROOT/dance_1723_video_10s" ;; motion_example_0506_demo_clip50) dst_dir="$FINAL_VIDEO_SUPPORT_ROOT/motion_example_0506_demo" ;; 2264_with_audio_clip50) dst_dir="$FINAL_VIDEO_SUPPORT_ROOT/2264_with_audio" ;; distilled_video_0226_clip50) dst_dir="$FINAL_VIDEO_SUPPORT_ROOT/distilled_video_0226" ;; esac "$PYTHON_BIN" - "$src_dir/metadata.jsonl" "$dst_dir/metadata.jsonl" <<'PY' import os, shutil, sys from pathlib import Path src, dst = map(Path, sys.argv[1:3]) dst.parent.mkdir(parents=True, exist_ok=True) tmp = dst.with_name(f".{dst.name}.{os.getpid()}.tmp") try: try: os.link(src, tmp) except OSError: shutil.copyfile(src, tmp) os.replace(tmp, dst) finally: try: tmp.unlink() except FileNotFoundError: pass PY done } if [[ $video_task_selected -eq 1 ]]; then copy_hidden_training_metadata fi write_support_config() { local support_config=$1 export PSI_PREP_SELECTED_SOURCES="$SELECTED_SOURCES" export PSI_PREP_SEED_ROOT="$SEED_ROOT" export PSI_PREP_QWEN3_TTS_ROOT="$QWEN3_TTS_ROOT" export PSI_PREP_SOURCE_SPLIT_ROOT="$SOURCE_SPLIT_ROOT" export PSI_PREP_FINAL_VIDEO_ROOT="$FINAL_VIDEO_SUPPORT_ROOT" export PSI_PREP_DANCE_AUDIO_WAV_ROOT="$DANCE_AUDIO_WAV_ROOT" export PSI_PREP_2264_AUDIO_WAV_ROOT="$AUDIO_2264_WAV_ROOT" export PSI_PREP_AUDIO_COMMAND_FULL_AUDIO="$AUDIO_COMMAND_FULL_AUDIO" export PSI_PREP_AUDIO_COMMAND_SAMPLE_STEP_FRAMES="$AUDIO_COMMAND_SAMPLE_STEP_FRAMES" export PSI_PREP_AUDIO_COMMAND_WINDOW_SECONDS="$AUDIO_COMMAND_WINDOW_SECONDS" "$PYTHON_BIN" - "$support_config" <<'PY' import json, os, sys from pathlib import Path selected = {value for value in os.environ["PSI_PREP_SELECTED_SOURCES"].split(",") if value} split_root = Path(os.environ["PSI_PREP_SOURCE_SPLIT_ROOT"]) video_root = Path(os.environ["PSI_PREP_FINAL_VIDEO_ROOT"]) qwen_root = Path(os.environ["PSI_PREP_QWEN3_TTS_ROOT"]) seed_root = Path(os.environ["PSI_PREP_SEED_ROOT"]) dance_wav_root = Path(os.environ["PSI_PREP_DANCE_AUDIO_WAV_ROOT"]) root_2264_wav = Path(os.environ["PSI_PREP_2264_AUDIO_WAV_ROOT"]) audio_command_full_audio = os.environ["PSI_PREP_AUDIO_COMMAND_FULL_AUDIO"] == "1" audio_command_step_frames = int(os.environ["PSI_PREP_AUDIO_COMMAND_SAMPLE_STEP_FRAMES"]) audio_command_window_seconds = float(os.environ["PSI_PREP_AUDIO_COMMAND_WINDOW_SECONDS"]) audio = [] if "psi0_dance_audio" in selected: audio.append({ "name": "psi0_dance_audio", "split_source_namespace": "dance_1723_video_10s", "dataset_root": str(dance_wav_root), "metadata_jsonl": str(dance_wav_root / "metadata.jsonl"), "motion_dir_key": "converted_path", "audio_path_key": "audio_path", "audio_file_extensions": ".wav", "records_cache_dir": str(split_root / "audio_records"), "dataset_name": "seed-audio-g1", }) if "2264_with_audio" in selected: audio.append({ "name": "2264_with_audio", "split_source_namespace": "2264_with_audio", "dataset_root": str(root_2264_wav), "metadata_jsonl": str(root_2264_wav / "metadata.jsonl"), "motion_dir_key": "converted_path", "audio_path_key": "audio_path", "audio_file_extensions": ".wav", "records_cache_dir": str(split_root / "audio_records"), "dataset_name": "2264-with-audio-g1", }) audio_command = [] if "qwen3_tts_seed_audio_command" in selected: audio_command.append({ "name": "qwen3_tts_seed_audio_command", "split_source_namespace": "seed", "root_dir": str(seed_root), "dataset_root": str(qwen_root), "metadata_jsonl": str(qwen_root / "metadata.jsonl"), "motion_dir_key": "filename", "audio_path_key": "audio_path", "instruction_key": "content_short_description", "motion_metadata_csv": str(seed_root / "metadata/seed_metadata_v004.csv"), "motion_metadata_key": "filename", "motion_metadata_path_key": "move_g1_path", "audio_file_extensions": ".wav", "records_cache_dir": str(split_root / "audio_command_records"), "audio_task": "command", "dataset_name": "seed-audio-command-g1", "full_audio": audio_command_full_audio, "sample_step_frames": audio_command_step_frames, "audio_window_seconds": audio_command_window_seconds, }) video = [] for source_name, namespace, directory, dataset_name in ( ("dance_1723_video_10s_clip50", "dance_1723_video_10s", "dance_1723_video_10s", "dance-video-clip-g1"), ("motion_example_0506_demo_clip50", "motion_example_0506_demo", "motion_example_0506_demo", "motion-example-video-clip-g1"), ("2264_with_audio_clip50", "2264_with_audio", "2264_with_audio", "2264-with-audio-video-clip-g1"), ): if source_name not in selected: continue root = video_root / directory video.append({ "name": source_name, "split_source_namespace": namespace, "dataset_root": str(root), "metadata_jsonl": "metadata.jsonl", "motion_dir_key": "converted_path", "video_path_key": "video_path", "instruction_key": "video_caption", "video_embed_dir": str(root / "embeds"), "records_cache_dir": str(split_root / "video_clip_records"), "dataset_name": dataset_name, }) payload = {"audio": audio, "audio_command": audio_command, "video_clip": video} path = Path(sys.argv[1]) path.parent.mkdir(parents=True, exist_ok=True) tmp = path.with_name(f".{path.name}.{os.getpid()}.tmp") tmp.write_text(json.dumps(payload, ensure_ascii=False, indent=2), encoding="utf-8") os.replace(tmp, path) PY } if [[ "$BUILD_SOURCE_SPLIT" == "1" ]]; then SUPPORT_CONFIG_FILE="$WORK_ROOT/source_split_sources.json" mkdir -p "$SOURCE_SPLIT_ROOT/runtime_indexes" write_support_config "$SUPPORT_CONFIG_FILE" AUDIO_SOURCES_JSON=$($PYTHON_BIN -c 'import json,sys; print(json.dumps(json.load(open(sys.argv[1]))["audio"],separators=(",",":")))' "$SUPPORT_CONFIG_FILE") AUDIO_COMMAND_SOURCES_JSON=$($PYTHON_BIN -c 'import json,sys; print(json.dumps(json.load(open(sys.argv[1]))["audio_command"],separators=(",",":")))' "$SUPPORT_CONFIG_FILE") VIDEO_CLIP_SOURCES_JSON=$($PYTHON_BIN -c 'import json,sys; print(json.dumps(json.load(open(sys.argv[1]))["video_clip"],separators=(",",":")))' "$SUPPORT_CONFIG_FILE") PREBUILD_TASKS=() csv_has "$SELECTED_TASKS" text && csv_has "$SELECTED_SOURCES" seed_text && PREBUILD_TASKS+=(text) if csv_has "$SELECTED_TASKS" audio_text || csv_has "$SELECTED_TASKS" audio_only; then PREBUILD_TASKS+=(audio); fi if csv_has "$SELECTED_TASKS" audio_command_text || csv_has "$SELECTED_TASKS" audio_command_only; then PREBUILD_TASKS+=(audio_command); fi [[ $video_task_selected -eq 1 ]] && PREBUILD_TASKS+=(video_clip) PREBUILD_TASKS_CSV=$(IFS=,; echo "${PREBUILD_TASKS[*]}") export PSI_OMNI_OSS_RUNTIME_INDEX_DIR="$SOURCE_SPLIT_ROOT/runtime_indexes" if [[ "$OUTPUT_KIND" == "hidden" ]]; then export PSI_PREBUILD_OMNI_OSS_KEY_INDEX=1 else # embeds 模式没有 hidden SQLite;这里只生成训练/验证 source split 记录。 export PSI_PREBUILD_OMNI_OSS_KEY_INDEX=0 fi export PSI_VIDEO_UNIQUE_RECORD_CACHE_FALLBACK=0 if [[ "$AUDIO_COMMAND_FULL_AUDIO" == "1" ]]; then AUDIO_COMMAND_FULL_AUDIO_ARG=--audio-command-full-audio else AUDIO_COMMAND_FULL_AUDIO_ARG=--no-audio-command-full-audio fi "$PYTHON_BIN" -u scripts/data/prebuild_multimodal_records_cache.py \ --tasks "$PREBUILD_TASKS_CSV" --splits train,val \ --action-chunk-size 30 --num-past-frames 0 --omni-hidden-root "$OUTPUT_ROOT" \ --text-root-dir "$SEED_ROOT" --text-metadata-csv metadata/seed_metadata_v004.csv \ --text-records-cache-dir "$SOURCE_SPLIT_ROOT/text_records" --text-split-source-namespace seed \ --audio-sources-json "$AUDIO_SOURCES_JSON" --audio-records-cache-dir "$SOURCE_SPLIT_ROOT/audio_records" \ --audio-sample-stride 1 --audio-sample-step-frames "$AUDIO_SAMPLE_STEP_FRAMES" \ --audio-motion-fps 50 --audio-audio-sampling-rate 16000 \ --audio-audio-window-seconds "$AUDIO_WINDOW_SECONDS" \ --audio-audio-left-context-seconds 0.0 --no-audio-full-audio \ --audio-command-sources-json "$AUDIO_COMMAND_SOURCES_JSON" \ --audio-command-records-cache-dir "$SOURCE_SPLIT_ROOT/audio_command_records" \ --audio-command-sample-stride 1 --audio-command-sample-step-frames "$AUDIO_COMMAND_SAMPLE_STEP_FRAMES" \ --audio-command-motion-fps 50 \ --audio-command-audio-sampling-rate 16000 --audio-command-audio-window-seconds "$AUDIO_COMMAND_WINDOW_SECONDS" \ --audio-command-audio-left-context-seconds 0.0 "$AUDIO_COMMAND_FULL_AUDIO_ARG" \ --video-sources-json '[]' \ --video-clip-sources-json "$VIDEO_CLIP_SOURCES_JSON" \ --video-clip-records-cache-dir "$SOURCE_SPLIT_ROOT/video_clip_records" \ --video-clip-sample-stride 1 --video-clip-motion-fps 50 \ --video-clip-video-sample-fps "$VIDEO_SAMPLE_FPS" \ --video-clip-video-size-pixels "$VIDEO_SIZE_PIXELS" --video-clip-video-decoder "$VIDEO_DECODER" \ --video-clip-record-build-workers 32 --video-clip-record-build-chunk-size 4096 \ --video-clip-trust-metadata --video-clip-action-exec-horizon 20 if [[ "$OUTPUT_KIND" == "hidden" ]]; then PREFLIGHT_ARGS=( scripts/data/prepare_omni_oss_training.py --oss-root "$OUTPUT_ROOT" --runtime-index-dir "$SOURCE_SPLIT_ROOT/runtime_indexes" --tasks "$SELECTED_TASKS" --splits train,val --val-ratio 0.1 --split-seed 42 ) for source in "${VIDEO_SOURCES[@]}"; do case "$source" in dance_1723_video_10s_clip50) namespace=dance_1723_video_10s; directory=dance_1723_video_10s ;; motion_example_0506_demo_clip50) namespace=motion_example_0506_demo; directory=motion_example_0506_demo ;; 2264_with_audio_clip50) namespace=2264_with_audio; directory=2264_with_audio ;; *) continue ;; esac PREFLIGHT_ARGS+=(--video-manifest-spec "$namespace=$FINAL_VIDEO_SUPPORT_ROOT/$directory/metadata.jsonl") done [[ $video_task_selected -eq 0 ]] && PREFLIGHT_ARGS+=(--skip-video-index) "$PYTHON_BIN" -u "${PREFLIGHT_ARGS[@]}" fi if [[ "$LIMIT" == "0" ]] \ && csv_has "$SELECTED_TASKS" text \ && csv_has "$SELECTED_TASKS" audio_text \ && csv_has "$SELECTED_TASKS" audio_command_text \ && csv_has "$SELECTED_TASKS" video_clip_text \ && csv_has "$SELECTED_SOURCES" seed_text \ && csv_has "$SELECTED_SOURCES" psi0_dance_audio \ && csv_has "$SELECTED_SOURCES" 2264_with_audio \ && csv_has "$SELECTED_SOURCES" qwen3_tts_seed_audio_command \ && csv_has "$SELECTED_SOURCES" dance_1723_video_10s_clip50 \ && csv_has "$SELECTED_SOURCES" motion_example_0506_demo_clip50 \ && csv_has "$SELECTED_SOURCES" 2264_with_audio_clip50; then "$PYTHON_BIN" -u scripts/data/audit_source_split_artifacts.py \ --cache-root "$SOURCE_SPLIT_ROOT" \ --runtime-index-dir "$SOURCE_SPLIT_ROOT/runtime_indexes" \ --video-root "$FINAL_VIDEO_SUPPORT_ROOT" fi "$PYTHON_BIN" scripts/data/manage_source_split_artifacts.py build-manifest --root "$SOURCE_SPLIT_ROOT" "$PYTHON_BIN" scripts/data/manage_source_split_artifacts.py verify --root "$SOURCE_SPLIT_ROOT" fi cleanup_video_resume_files() { local source root for source in "${VIDEO_SOURCES[@]}"; do root=$(video_output_dir "$source") [[ "$root" == "$VIDEO_CACHE_ROOT"/* ]] || { echo "拒绝清理意外路径: $root" >&2; return 1; } rm -rf -- "$root/.source_done" find "$root" -maxdepth 1 -type f \( -name 'metadata.shard-*.jsonl' -o -name '.*.tmp' \) -delete if [[ -d "$root/embeds" ]]; then find "$root/embeds" -maxdepth 1 -type f \( -name 'resume*.json' -o -name 'manifest.shard-*.json' -o -name '*.lock' -o -name '.*.tmp' \) -delete fi done } if [[ "$HIDDEN_LAYOUT" == "source" && "$LIMIT" != "0" ]]; then COMPLETE_PATH="$OUTPUT_ROOT/_reports/prepare_debug_limit_${LIMIT}.json" else COMPLETE_PATH="$OUTPUT_ROOT/_reports/prepare_complete.json" fi mkdir -p "$(dirname "$COMPLETE_PATH")" "$PYTHON_BIN" - "$COMPLETE_PATH" "$OUTPUT_KIND" "$OUTPUT_ROOT" "$SELECTED_TASKS" "$SELECTED_SOURCES" \ "$BUILD_SOURCE_SPLIT" "$MODEL_PATH" "$CUDA_VISIBLE_DEVICES" "$WORKERS_PER_GPU" \ "$HIDDEN_BATCH_SIZE" "$AUDIO_FRAME_COUNT_SOURCE" "$HIDDEN_SHARD_SIZE" \ "$FINAL_SAMPLES_PER_SHARD" "$FINAL_MAX_SHARD_GIB" "$SHUFFLE_SEED" \ "$AUDIO_WAV_ROOT" "$AUDIO_SAMPLE_STEP_FRAMES" "$AUDIO_WINDOW_SECONDS" \ "$AUDIO_COMMAND_FULL_AUDIO" "$AUDIO_COMMAND_SAMPLE_STEP_FRAMES" "$AUDIO_COMMAND_WINDOW_SECONDS" \ "$HIDDEN_LAYOUT" "$LIMIT" "$ONLINE_VERIFY_SOURCES" <<'PY' import json, os, sys, time from pathlib import Path path = Path(sys.argv[1]) payload = { "format": "psi0_unified_preprocess_v4", "output_kind": sys.argv[2], "output_root": str(Path(sys.argv[3]).resolve()), "tasks": [value for value in sys.argv[4].split(",") if value], "sources": [value for value in sys.argv[5].split(",") if value], "source_split_ready": sys.argv[6] == "1", "model_path": str(Path(sys.argv[7]).resolve()), "cuda_visible_devices": sys.argv[8], "workers_per_gpu": int(sys.argv[9]), "hidden_batch_size": int(sys.argv[10]), "audio_frame_count_source": sys.argv[11], "hidden_layout": sys.argv[22], "limit_sources": int(sys.argv[23]) if sys.argv[22] == "source" else 0, "online_verify_sources": int(sys.argv[24]) if sys.argv[22] == "source" else 0, "hidden_staging_shard_size": int(sys.argv[12]) if sys.argv[22] == "oss" else None, "final_samples_per_shard": int(sys.argv[13]) if sys.argv[22] == "oss" else None, "final_max_shard_gib": float(sys.argv[14]) if sys.argv[22] == "oss" else None, "shuffle_seed": int(sys.argv[15]), "audio_input_format": "wav", "audio_wav_root": str(Path(sys.argv[16]).resolve()), "audio_sample_step_frames": int(sys.argv[17]), "audio_window_seconds": float(sys.argv[18]), "audio_command_full_audio": sys.argv[19] == "1", "audio_command_sample_step_frames": int(sys.argv[20]), "audio_command_window_seconds": float(sys.argv[21]), "completed_at": time.time(), } if sys.argv[22] == "source": verify_path = Path(sys.argv[3]) / "_reports" / "online_hidden_verify.json" verification = json.loads(verify_path.read_text(encoding="utf-8")) if ( int(verification.get("verified_windows", 0)) <= 0 or int(verification.get("mismatched_windows", -1)) != 0 or float(verification.get("max_abs_error", -1.0)) != 0.0 ): raise RuntimeError(f"Invalid exact-online hidden verification: {verify_path}") payload["online_hidden_verification"] = verification tmp = path.with_name(f".{path.name}.{os.getpid()}.tmp") tmp.write_text(json.dumps(payload, ensure_ascii=False, indent=2), encoding="utf-8") os.replace(tmp, path) PY cleanup_video_resume_files rm -rf -- "$INDEX_TMP_DIR" if [[ "$CLEAN_STAGING_AFTER_SUCCESS" == "1" ]]; then "$PYTHON_BIN" - "$WORK_ROOT" "$OUTPUT_ROOT" <<'PY' import json, shutil, sys from pathlib import Path work = Path(sys.argv[1]).resolve() output = Path(sys.argv[2]).resolve() marker = work / ".psi0_preprocess_work.json" if work == output or not marker.is_file(): raise RuntimeError(f"Refusing to remove unowned staging directory: {work}") payload = json.loads(marker.read_text(encoding="utf-8")) if payload.get("format") != "psi0_preprocess_work_v1" or Path(payload.get("output_root", "")).resolve() != output: raise RuntimeError(f"Invalid staging ownership marker: {marker}") shutil.rmtree(work) print(f"[cleanup] removed staging: {work}") PY fi echo "[完成] 输出已生成并验证: $OUTPUT_ROOT"
#!/usr/bin/env bash
set -Eeuo pipefail
# Archived data builder; keep relative dependencies rooted at the HFD project.
PROJECT_ROOT=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)
cd "$PROJECT_ROOT"
# =============================================================================
# 只需要改这一段
# =============================================================================
# 默认只生成 HFD audio-command 需要的 hidden:一个完整 WAV/source
# 对应一个紧凑 safetensors,内部仍是 0.6 秒窗口、0.4 秒滑动。
# 不生成逐窗口文件,不做大分片 staging/merge,也不构造无关任务。
# 旧的多任务 OSS 大分片流程可显式设置 HIDDEN_LAYOUT=oss 后继续复用。
OUTPUT_KIND="hidden" # embeds | hidden
OUTPUT_ROOT="/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/videogen/hfd_audio_command_source"
HIDDEN_LAYOUT="source" # source | oss
# 统一音频入口是 WAV。Qwen3-TTS 直接读取原始 seed_audio/*.wav;dance/2264
# 原始数据只有 MP4 音轨,因此先把历史 manifest 的规范采样(NPY/少量 MP4 fallback)
# 无损封装成 float32 WAV。
# 这样既不改变旧 hidden 的数值,又能让后续音频切 clip 只面对一种输入格式。
AUDIO_WAV_ROOT="/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/videogen/psi0_audio_wav_v26"
# 可选任务:text,audio_text,audio_only,audio_command_text,audio_command_only,
# video_clip_text,video_clip_only
SELECTED_TASKS="audio_command_text"
# 可选数据源:
# seed_text
# psi0_dance_audio
# 2264_with_audio
# qwen3_tts_seed_audio_command
# dance_1723_video_10s_clip50
# motion_example_0506_demo_clip50
# 2264_with_audio_clip50
# distilled_video_0226_clip50(目前仅建议用于 embeds;正式训练配置尚未引用它)
SELECTED_SOURCES="qwen3_tts_seed_audio_command"
CUDA_VISIBLE_DEVICES="${CUDA_VISIBLE_DEVICES:-0,1,2,3,4,5,6,7}" # embeds 模式不用 GPU
WORKERS_PER_GPU=8
TORCHRUN_LOCAL_ADDR="127.0.0.1"
BUILD_SOURCE_SPLIT=0 # HFD source hidden 不需要旧多任务 split 包
CLEAN_STAGING_AFTER_SUCCESS=1 # hidden 成功后删除中间大 shard 和 video embeds
OVERWRITE=0
# 下面通常不用改。正式数据必须保持 LIMIT=0;非零只用于逐阶段调试,
# 其输出不应直接交给训练。
LIMIT=0
VIDEO_METADATA_SHARDS=4
VIDEO_BUILD_WORKERS=64
VIDEO_EMBED_WORKERS=32
VIDEO_SAMPLE_FPS=8
VIDEO_SIZE_PIXELS=50176
VIDEO_DECODER="ffmpeg"
AUDIO_WAV_WORKERS=32
AUDIO_SAMPLE_STEP_FRAMES=1
AUDIO_WINDOW_SECONDS=0.6
# audio_command 默认不再输入整段 WAV:只保留完整的 0.6 秒窗口,
# 每 20 个 50 FPS 动作帧(0.4 秒)向后滑动一次。
AUDIO_COMMAND_FULL_AUDIO=0 # 0:切片 | 1:旧版整段语音
AUDIO_COMMAND_SAMPLE_STEP_FRAMES=20
AUDIO_COMMAND_WINDOW_SECONDS=0.6
HIDDEN_BATCH_SIZE=1
HIDDEN_SHARD_SIZE=4096
# metadata:严格复现旧 cache_omni_hidden_states_v23.sh 的样本范围。
# motion_csv:按动作 CSV 的真实帧数生成,属于新的数据定义。
AUDIO_FRAME_COUNT_SOURCE="metadata" # metadata | motion_csv
FINAL_SAMPLES_PER_SHARD=4096
FINAL_MAX_SHARD_GIB=4.5
SHUFFLE_SEED=20260710
MERGE_WORKERS=32
MERGE_SOURCE_READER_CACHE=32
VERIFY_SAMPLES=64
ONLINE_VERIFY_SOURCES=64 # 前若干完整 WAV 逐窗口复算,要求与在线 B=1 逐元素相等
VERIFY_CANONICAL_COUNTS=1 # 默认全量四任务时核对当前数据定义的精确条数
MANAGE_LOCAL_GPU_MONITOR=1
GPU_MONITOR_SCRIPT="/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/monitor/cmd.sh"
# 自动化/小测试可以用 PSI_PREP_* 临时覆盖;正常使用无需传任何参数。
OUTPUT_KIND=${PSI_PREP_OUTPUT_KIND:-$OUTPUT_KIND}
OUTPUT_ROOT=${PSI_PREP_OUTPUT_ROOT:-$OUTPUT_ROOT}
HIDDEN_LAYOUT=${PSI_PREP_HIDDEN_LAYOUT:-$HIDDEN_LAYOUT}
AUDIO_WAV_ROOT=${PSI_PREP_AUDIO_WAV_ROOT:-$AUDIO_WAV_ROOT}
SELECTED_TASKS=${PSI_PREP_TASKS:-$SELECTED_TASKS}
SELECTED_SOURCES=${PSI_PREP_SOURCES:-$SELECTED_SOURCES}
CUDA_VISIBLE_DEVICES=${PSI_PREP_CUDA_VISIBLE_DEVICES:-$CUDA_VISIBLE_DEVICES}
WORKERS_PER_GPU=${PSI_PREP_WORKERS_PER_GPU:-$WORKERS_PER_GPU}
TORCHRUN_LOCAL_ADDR=${PSI_PREP_TORCHRUN_LOCAL_ADDR:-$TORCHRUN_LOCAL_ADDR}
BUILD_SOURCE_SPLIT=${PSI_PREP_BUILD_SOURCE_SPLIT:-$BUILD_SOURCE_SPLIT}
CLEAN_STAGING_AFTER_SUCCESS=${PSI_PREP_CLEAN_STAGING:-$CLEAN_STAGING_AFTER_SUCCESS}
OVERWRITE=${PSI_PREP_OVERWRITE:-$OVERWRITE}
LIMIT=${PSI_PREP_LIMIT:-$LIMIT}
VIDEO_METADATA_SHARDS=${PSI_PREP_VIDEO_METADATA_SHARDS:-$VIDEO_METADATA_SHARDS}
VIDEO_BUILD_WORKERS=${PSI_PREP_VIDEO_BUILD_WORKERS:-$VIDEO_BUILD_WORKERS}
VIDEO_EMBED_WORKERS=${PSI_PREP_VIDEO_EMBED_WORKERS:-$VIDEO_EMBED_WORKERS}
AUDIO_WAV_WORKERS=${PSI_PREP_AUDIO_WAV_WORKERS:-$AUDIO_WAV_WORKERS}
AUDIO_SAMPLE_STEP_FRAMES=${PSI_PREP_AUDIO_SAMPLE_STEP_FRAMES:-$AUDIO_SAMPLE_STEP_FRAMES}
AUDIO_WINDOW_SECONDS=${PSI_PREP_AUDIO_WINDOW_SECONDS:-$AUDIO_WINDOW_SECONDS}
AUDIO_COMMAND_FULL_AUDIO=${PSI_PREP_AUDIO_COMMAND_FULL_AUDIO:-$AUDIO_COMMAND_FULL_AUDIO}