#!/usr/bin/env bash set -euo pipefail cd /cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/Psi0-yth-v18 PSI_PYTHON_BIN=${PSI_PYTHON_BIN:-} PSI_PYTHON_ENV=${PSI_PYTHON_ENV:-} if [ -z "$PSI_PYTHON_BIN" ] && [ -n "$PSI_PYTHON_ENV" ]; then if [ ! -x "$PSI_PYTHON_ENV/bin/python" ]; then echo "ERROR: PSI_PYTHON_ENV does not contain an executable python: $PSI_PYTHON_ENV/bin/python" >&2 exit 2 fi source "$PSI_PYTHON_ENV/bin/activate" fi [ -f .env ] && source .env export PYTHONPATH=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/Psi0-yth-v18/src export CUDA_VISIBLE_DEVICES=${CUDA_VISIBLE_DEVICES:-0} export OMP_NUM_THREADS=${OMP_NUM_THREADS:-1} PYTHON_BIN=${PSI_PYTHON_BIN:-$(command -v python)} if [ ! -x "$PYTHON_BIN" ]; then echo "ERROR: python executable is not valid: $PYTHON_BIN" >&2 exit 2 fi DATASET_ROOT=${PSI_VIDEO_CLIP_DATASET_ROOT:-/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/videogen/psi0_video_clip_50f_rand8} DANCE_CLIP_ROOT=$DATASET_ROOT/dance_1723_video_10s MOTION_EXAMPLE_CLIP_ROOT=$DATASET_ROOT/motion_example_0506_demo DANCE_CLIP_EMBED_ROOT=${PSI_DANCE_VIDEO_CLIP_EMBED_ROOT:-$DANCE_CLIP_ROOT/embeds} MOTION_EXAMPLE_CLIP_EMBED_ROOT=${PSI_MOTION_EXAMPLE_VIDEO_CLIP_EMBED_ROOT:-$MOTION_EXAMPLE_CLIP_ROOT/embeds} DANCE_MOTION_ROOT=${PSI_DANCE_MOTION_ROOT:-/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/GR00T-WholeBodyControl/gear_sonic_deploy/reference/converted/psi0_dance/gvhmr} DANCE_VIDEO_ROOT=${PSI_DANCE_VIDEO_ROOT:-/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/dance_1723_video_psi0/dance_1723_video_10s} MOTION_EXAMPLE_ROOT=${PSI_VIDEO_DEMO_ROOT:-/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/videogen/motion_example_0506_demo} MODEL_PATH=${PSI_PRETRAIN_BACKBONE_PATH:-/cpfs01/VideoGen/public_ckpt/Qwen/Qwen2.5-Omni-3B} VIDEO_SAMPLE_FPS=${VIDEO_SAMPLE_FPS:-32} VIDEO_SIZE_PIXELS=${VIDEO_SIZE_PIXELS:-50176} VIDEO_DECODER=${VIDEO_DECODER:-ffmpeg} CLIP_FRAMES=${CLIP_FRAMES:-50} ACTION_CHUNK_FRAMES=${ACTION_CHUNK_FRAMES:-30} CHUNK_STRIDE_FRAMES=${CHUNK_STRIDE_FRAMES:-20} MOTION_FPS=${MOTION_FPS:-50} CLIPS_PER_VIDEO=${CLIPS_PER_VIDEO:-2} CLIP_SELECTION_MODE=${CLIP_SELECTION_MODE:-middle_random} CLIP_RANDOM_SEED=${CLIP_RANDOM_SEED:-42} CLIP_MIDDLE_FRACTION=${CLIP_MIDDLE_FRACTION:-0.8} CHUNK_PROGRESS_INTERVAL=${CHUNK_PROGRESS_INTERVAL:-0} BUILD_NUM_WORKERS=${BUILD_NUM_WORKERS:-32} CHUNK_NUM_WORKERS=${CHUNK_NUM_WORKERS:-1} FFMPEG_THREADS=${FFMPEG_THREADS:-1} X264_PRESET=${X264_PRESET:-} EXISTING_VIDEO_CHECK=${EXISTING_VIDEO_CHECK:-ffprobe} EMBED_NUM_WORKERS=${EMBED_NUM_WORKERS:-64} EMBED_SAVE_COMPRESSED=${EMBED_SAVE_COMPRESSED:-0} EXISTING_CACHE_CHECK=${EXISTING_CACHE_CHECK:-strict} BUILD_CLIPS=${BUILD_CLIPS:-1} CACHE_EMBEDS=${CACHE_EMBEDS:-1} BUILD_DANCE=${BUILD_DANCE:-1} BUILD_MOTION_EXAMPLE=${BUILD_MOTION_EXAMPLE:-1} CACHE_DANCE=${CACHE_DANCE:-1} CACHE_MOTION_EXAMPLE=${CACHE_MOTION_EXAMPLE:-1} LIMIT=${LIMIT:-0} OVERWRITE_CLIPS=${OVERWRITE_CLIPS:-0} OVERWRITE_EMBEDS=${OVERWRITE_EMBEDS:-0} SHARD_NUM=${SHARD_NUM:-1} SHARD_ID=${SHARD_ID:-0} METADATA_ONLY=${METADATA_ONLY:-0} DIRECT_SOURCE_CLIPS=${DIRECT_SOURCE_CLIPS:-0} if ! [[ "$SHARD_NUM" =~ ^[0-9]+$ ]] || ! [[ "$SHARD_ID" =~ ^[0-9]+$ ]]; then echo "SHARD_NUM and SHARD_ID must be non-negative integers" >&2 exit 2 fi if (( SHARD_NUM <= 0 )); then echo "SHARD_NUM must be positive" >&2 exit 2 fi if (( SHARD_ID >= SHARD_NUM )); then echo "SHARD_ID must satisfy 0 <= SHARD_ID < SHARD_NUM" >&2 exit 2 fi printf -v SHARD_ID_PAD "%05d" "$SHARD_ID" printf -v SHARD_NUM_PAD "%05d" "$SHARD_NUM" clip_overwrite_args=() if [ "$OVERWRITE_CLIPS" = "1" ]; then clip_overwrite_args+=(--overwrite) fi embed_overwrite_args=() if [ "$OVERWRITE_EMBEDS" = "1" ]; then embed_overwrite_args+=(--overwrite) fi embed_compress_args=() if [ "$EMBED_SAVE_COMPRESSED" = "1" ]; then embed_compress_args+=(--save-compressed) fi limit_args=() if [ "$LIMIT" -gt 0 ]; then limit_args+=(--limit "$LIMIT") fi shard_args=() if (( SHARD_NUM > 1 )); then shard_args+=(--shard-num "$SHARD_NUM" --shard-id "$SHARD_ID") fi chunk_progress_args=() if [ "$CHUNK_PROGRESS_INTERVAL" -gt 0 ]; then chunk_progress_args+=(--chunk-progress-interval "$CHUNK_PROGRESS_INTERVAL") fi metadata_only_args=() if [ "$METADATA_ONLY" = "1" ]; then metadata_only_args+=(--metadata-only) fi direct_source_clip_args=() if [ "$DIRECT_SOURCE_CLIPS" = "1" ]; then direct_source_clip_args+=(--direct-source-clips) fi cache_metadata_jsonl() { local clip_root="$1" local shard_metadata="metadata.shard-${SHARD_ID_PAD}-of-${SHARD_NUM_PAD}.jsonl" if (( SHARD_NUM > 1 )) && [ -f "$clip_root/$shard_metadata" ]; then printf "%s" "$shard_metadata" else printf "%s" "metadata.jsonl" fi } echo "Dataset root: $DATASET_ROOT" echo "Dance clip embed root: $DANCE_CLIP_EMBED_ROOT" echo "Motion example clip embed root: $MOTION_EXAMPLE_CLIP_EMBED_ROOT" echo "Build clips: $BUILD_CLIPS" echo "Cache embeds: $CACHE_EMBEDS" echo "Build dance: $BUILD_DANCE" echo "Build motion_example: $BUILD_MOTION_EXAMPLE" echo "Cache dance: $CACHE_DANCE" echo "Cache motion_example: $CACHE_MOTION_EXAMPLE" echo "Clip frames: $CLIP_FRAMES" echo "Action chunk frames: $ACTION_CHUNK_FRAMES" echo "Chunk stride frames: $CHUNK_STRIDE_FRAMES" echo "Clips per video: $CLIPS_PER_VIDEO" echo "Clip selection mode: $CLIP_SELECTION_MODE" echo "Clip random seed: $CLIP_RANDOM_SEED" echo "Clip middle fraction: $CLIP_MIDDLE_FRACTION" echo "Chunk progress interval: $CHUNK_PROGRESS_INTERVAL" echo "Build workers: $BUILD_NUM_WORKERS" echo "Chunk workers per source: $CHUNK_NUM_WORKERS" echo "FFmpeg threads: $FFMPEG_THREADS" echo "x264 preset: ${X264_PRESET:-}" echo "Existing video check: $EXISTING_VIDEO_CHECK" echo "Embed workers: $EMBED_NUM_WORKERS" echo "Embed compressed: $EMBED_SAVE_COMPRESSED" echo "Existing cache check: $EXISTING_CACHE_CHECK" echo "Video sample fps: $VIDEO_SAMPLE_FPS" echo "Video size pixels: $VIDEO_SIZE_PIXELS" echo "Shard: $SHARD_ID/$SHARD_NUM" echo "Metadata only: $METADATA_ONLY" echo "Direct source clips: $DIRECT_SOURCE_CLIPS" echo "Python: $PYTHON_BIN" if [ "$CACHE_EMBEDS" = "1" ]; then "$PYTHON_BIN" - <<'PY' import sys from transformers import Qwen2_5OmniProcessor print("Python executable:", sys.executable) print("Qwen2_5OmniProcessor import: ok") PY fi if [ "$BUILD_CLIPS" = "1" ] && [ "$BUILD_DANCE" = "1" ]; then "$PYTHON_BIN" scripts/data/build_video_clip_dataset.py \ --dataset dance \ --output-root "$DANCE_CLIP_ROOT" \ --dance-motion-root "$DANCE_MOTION_ROOT" \ --dance-video-root "$DANCE_VIDEO_ROOT" \ --clip-frames "$CLIP_FRAMES" \ --action-chunk-frames "$ACTION_CHUNK_FRAMES" \ --chunk-stride-frames "$CHUNK_STRIDE_FRAMES" \ --motion-fps "$MOTION_FPS" \ --clips-per-video "$CLIPS_PER_VIDEO" \ --clip-selection-mode "$CLIP_SELECTION_MODE" \ --random-seed "$CLIP_RANDOM_SEED" \ --middle-fraction "$CLIP_MIDDLE_FRACTION" \ --num-workers "$BUILD_NUM_WORKERS" \ --chunk-workers "$CHUNK_NUM_WORKERS" \ --ffmpeg-threads "$FFMPEG_THREADS" \ --x264-preset "$X264_PRESET" \ --existing-video-check "$EXISTING_VIDEO_CHECK" \ "${shard_args[@]}" \ "${chunk_progress_args[@]}" \ "${limit_args[@]}" \ "${metadata_only_args[@]}" \ "${clip_overwrite_args[@]}" fi if [ "$BUILD_CLIPS" = "1" ] && [ "$BUILD_MOTION_EXAMPLE" = "1" ]; then "$PYTHON_BIN" scripts/data/build_video_clip_dataset.py \ --dataset motion_example \ --output-root "$MOTION_EXAMPLE_CLIP_ROOT" \ --motion-example-root "$MOTION_EXAMPLE_ROOT" \ --clip-frames "$CLIP_FRAMES" \ --action-chunk-frames "$ACTION_CHUNK_FRAMES" \ --chunk-stride-frames "$CHUNK_STRIDE_FRAMES" \ --motion-fps "$MOTION_FPS" \ --clips-per-video "$CLIPS_PER_VIDEO" \ --clip-selection-mode "$CLIP_SELECTION_MODE" \ --random-seed "$CLIP_RANDOM_SEED" \ --middle-fraction "$CLIP_MIDDLE_FRACTION" \ --num-workers "$BUILD_NUM_WORKERS" \ --chunk-workers "$CHUNK_NUM_WORKERS" \ --ffmpeg-threads "$FFMPEG_THREADS" \ --x264-preset "$X264_PRESET" \ --existing-video-check "$EXISTING_VIDEO_CHECK" \ "${shard_args[@]}" \ "${chunk_progress_args[@]}" \ "${limit_args[@]}" \ "${metadata_only_args[@]}" \ "${clip_overwrite_args[@]}" fi if [ "$CACHE_EMBEDS" = "1" ] && [ "$CACHE_DANCE" = "1" ]; then dance_metadata_jsonl=$(cache_metadata_jsonl "$DANCE_CLIP_ROOT") dance_cache_shard_args=("${shard_args[@]}") if (( SHARD_NUM > 1 )) && [ "$dance_metadata_jsonl" != "metadata.jsonl" ]; then dance_cache_shard_args+=(--input-already-sharded) fi "$PYTHON_BIN" cache_video_omni_embeds.py \ --dataset-root "$DANCE_CLIP_ROOT" \ --metadata-jsonl "$dance_metadata_jsonl" \ --embed-root "$DANCE_CLIP_EMBED_ROOT" \ --model-path "$MODEL_PATH" \ --video-sample-fps "$VIDEO_SAMPLE_FPS" \ --video-size-pixels "$VIDEO_SIZE_PIXELS" \ --video-decoder "$VIDEO_DECODER" \ --dtype float16 \ --num-workers "$EMBED_NUM_WORKERS" \ --existing-cache-check "$EXISTING_CACHE_CHECK" \ "${dance_cache_shard_args[@]}" \ "${limit_args[@]}" \ "${direct_source_clip_args[@]}" \ "${embed_compress_args[@]}" \ "${embed_overwrite_args[@]}" fi if [ "$CACHE_EMBEDS" = "1" ] && [ "$CACHE_MOTION_EXAMPLE" = "1" ]; then motion_metadata_jsonl=$(cache_metadata_jsonl "$MOTION_EXAMPLE_CLIP_ROOT") motion_cache_shard_args=("${shard_args[@]}") if (( SHARD_NUM > 1 )) && [ "$motion_metadata_jsonl" != "metadata.jsonl" ]; then motion_cache_shard_args+=(--input-already-sharded) fi "$PYTHON_BIN" cache_video_omni_embeds.py \ --dataset-root "$MOTION_EXAMPLE_CLIP_ROOT" \ --metadata-jsonl "$motion_metadata_jsonl" \ --embed-root "$MOTION_EXAMPLE_CLIP_EMBED_ROOT" \ --model-path "$MODEL_PATH" \ --video-sample-fps "$VIDEO_SAMPLE_FPS" \ --video-size-pixels "$VIDEO_SIZE_PIXELS" \ --video-decoder "$VIDEO_DECODER" \ --dtype float16 \ --num-workers "$EMBED_NUM_WORKERS" \ --existing-cache-check "$EXISTING_CACHE_CHECK" \ "${motion_cache_shard_args[@]}" \ "${limit_args[@]}" \ "${direct_source_clip_args[@]}" \ "${embed_compress_args[@]}" \ "${embed_overwrite_args[@]}" fi echo "[DONE] video clip datasets and missing caches are ready."