Download infer_render_auto.sh from Tinghong-Ye22/code-Psi0-yth-v21: direct link, hf CLI and curl.
- Browser
- Download file 25.6 kB
-
https://huggingface.co/Tinghong-Ye22/code-Psi0-yth-v21/resolve/main/infer_render_auto.sh
- Command line
-
hf download hf://Tinghong-Ye22/code-Psi0-yth-v21/infer_render_auto.sh
-
curl -L -o infer_render_auto.sh https://huggingface.co/Tinghong-Ye22/code-Psi0-yth-v21/resolve/main/infer_render_auto.sh
25.6 kB
| REPO_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/Psi0-yth-v21 | |
| DEFAULT_RUN_DIR=$REPO_ROOT/runs/posttrain_v1_trimodal/posttrain_actionhead_M.sg.flow1000.cwml.lr1.0e-04.b256.gpus8.2606241549 | |
| cd "$REPO_ROOT" | |
| export PYTHONPATH="$REPO_ROOT/src" | |
| source /cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/Psi0-yth-v15/.venv-psi/bin/activate | |
| [ -f .env ] && source .env | |
| export PYTHONPATH="$REPO_ROOT/src" | |
| export CUDA_VISIBLE_DEVICES=${CUDA_VISIBLE_DEVICES:-0} | |
| export PSI_INFER_FLOAT32=1 | |
| pkill -f monitor.py || true | |
| canonical_mode() { | |
| case "$1" in | |
| all|multi|full) | |
| echo all | |
| ;; | |
| text|pure_text) | |
| echo text | |
| ;; | |
| audio|audio_text|audio_only|audio_pure) | |
| echo audio | |
| ;; | |
| audio_command|audio_command_text|audio_command_only|command_audio) | |
| echo audio_command | |
| ;; | |
| video_clip|video_clip_text|video_clip_only) | |
| echo video_clip | |
| ;; | |
| video|video_text|video_only) | |
| echo video | |
| ;; | |
| *) | |
| echo "$1" | |
| ;; | |
| esac | |
| } | |
| RAW_MODE=${PSI_AUTO_INFER_MODE:-all} | |
| MODE=$(canonical_mode "$RAW_MODE") | |
| MODE_SEQUENCE=(text audio audio_command video_clip) | |
| # MODE_SEQUENCE=(audio_command) | |
| RUNS_ROOT=${RUNS_ROOT:-$REPO_ROOT/runs/posttrain_v1_trimodal} | |
| INFER_PY=${INFER_PY:-/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/Psi0-yth-v15/.venv-psi/bin/python} | |
| # RUN_DIR=${RUN_DIR:-auto} | |
| RUN_DIR=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/Psi0-yth-v21/runs/posttrain_v1_trimodal/posttrain.sg.flow1000.cwml.lr1.0e-04.b512.gpus8.2607051648 | |
| if [ $# -ge 1 ]; then | |
| RUN_DIR=$1 | |
| elif [ "$RUN_DIR" = "auto" ] || [ -z "$RUN_DIR" ]; then | |
| AUTO_RUN_DIR=$("$INFER_PY" - "$MODE" "$RUNS_ROOT" <<'PY' | |
| import json | |
| import re | |
| import sys | |
| from pathlib import Path | |
| mode = sys.argv[1] | |
| runs_root = Path(sys.argv[2]) | |
| def ratio(data, key): | |
| try: | |
| return float(data.get(key, 0.0)) | |
| except Exception: | |
| return 0.0 | |
| def supported_modes(run_config): | |
| data = run_config.get("data", {}) | |
| modes = [] | |
| if ratio(data, "text_ratio") > 0 or "root_dir" in data: | |
| modes.append("text") | |
| if ratio(data, "audio_ratio") > 0 or ratio(data, "audio_only_ratio") > 0: | |
| modes.append("audio") | |
| if ratio(data, "audio_command_ratio") > 0 or ratio(data, "audio_command_only_ratio") > 0: | |
| modes.append("audio_command") | |
| if ratio(data, "video_clip_text_ratio") > 0 or ratio(data, "video_clip_only_ratio") > 0: | |
| modes.append("video_clip") | |
| if ratio(data, "video_ratio") > 0 or ratio(data, "video_only_ratio") > 0: | |
| modes.append("video") | |
| return modes | |
| def run_sort_key(path): | |
| match = re.search(r"(\d{10})$", path.name) | |
| if match: | |
| return int(match.group(1)) | |
| return int(path.stat().st_mtime) | |
| candidates = [] | |
| for path in runs_root.glob("posttrain*.gpus8.*"): | |
| config_path = path / "run_config.json" | |
| if not config_path.exists(): | |
| continue | |
| try: | |
| run_config = json.loads(config_path.read_text()) | |
| except Exception: | |
| continue | |
| modes = supported_modes(run_config) | |
| if mode == "all": | |
| score = len([m for m in ("text", "audio", "audio_command", "video_clip", "video") if m in modes]) | |
| else: | |
| score = 1 if mode in modes else 0 | |
| if score > 0: | |
| candidates.append((score, run_sort_key(path), path)) | |
| if candidates: | |
| candidates.sort() | |
| print(str(candidates[-1][2])) | |
| PY | |
| ) | |
| if [ -n "$AUTO_RUN_DIR" ]; then | |
| RUN_DIR=$AUTO_RUN_DIR | |
| else | |
| RUN_DIR=$DEFAULT_RUN_DIR | |
| fi | |
| fi | |
| if [ -z "$RUN_DIR" ] || [ ! -f "$RUN_DIR/run_config.json" ]; then | |
| echo "ERROR: cannot resolve a valid RUN_DIR for MODE=$MODE" | |
| echo " Pass it explicitly: bash infer_render_auto.sh /path/to/run_dir" | |
| exit 1 | |
| fi | |
| SUPPORTED_MODES_CSV=$("$INFER_PY" - "$RUN_DIR" <<'PY' | |
| import json | |
| import sys | |
| from pathlib import Path | |
| run_dir = Path(sys.argv[1]) | |
| config = json.loads((run_dir / "run_config.json").read_text()) | |
| data = config.get("data", {}) | |
| def ratio(key): | |
| try: | |
| return float(data.get(key, 0.0)) | |
| except Exception: | |
| return 0.0 | |
| supported = [] | |
| if ratio("text_ratio") > 0 or "root_dir" in data: | |
| supported.append("text") | |
| if ratio("audio_ratio") > 0 or ratio("audio_only_ratio") > 0: | |
| supported.append("audio") | |
| if ratio("audio_command_ratio") > 0 or ratio("audio_command_only_ratio") > 0: | |
| supported.append("audio_command") | |
| if ratio("video_clip_text_ratio") > 0 or ratio("video_clip_only_ratio") > 0: | |
| supported.append("video_clip") | |
| if ratio("video_ratio") > 0 or ratio("video_only_ratio") > 0: | |
| supported.append("video") | |
| print(",".join(supported)) | |
| PY | |
| ) | |
| if [ -z "$SUPPORTED_MODES_CSV" ]; then | |
| echo "ERROR: RUN_DIR does not expose any supported inference modes: $RUN_DIR" | |
| exit 1 | |
| fi | |
| IFS=',' read -r -a SUPPORTED_MODES <<< "$SUPPORTED_MODES_CSV" | |
| mode_supported() { | |
| local wanted | |
| local supported | |
| wanted=$(canonical_mode "$1") | |
| for supported in "${SUPPORTED_MODES[@]}"; do | |
| if [ "$supported" = "$wanted" ]; then | |
| return 0 | |
| fi | |
| done | |
| return 1 | |
| } | |
| TARGET_MODES=() | |
| if [ "$MODE" = "all" ]; then | |
| if mode_supported text; then | |
| TARGET_MODES+=(text) | |
| fi | |
| if mode_supported audio; then | |
| TARGET_MODES+=(audio) | |
| fi | |
| if mode_supported audio_command; then | |
| TARGET_MODES+=(audio_command) | |
| fi | |
| if mode_supported video_clip; then | |
| TARGET_MODES+=(video_clip) | |
| elif mode_supported video; then | |
| TARGET_MODES+=(video) | |
| fi | |
| if [ ${#TARGET_MODES[@]} -eq 0 ]; then | |
| echo "ERROR: MODE=all but RUN_DIR has no supported target modes: $RUN_DIR" | |
| exit 1 | |
| fi | |
| else | |
| if ! mode_supported "$RAW_MODE"; then | |
| echo "ERROR: RUN_DIR does not look compatible with MODE=$RAW_MODE: $RUN_DIR" | |
| echo " supported modes: ${SUPPORTED_MODES[*]}" | |
| exit 1 | |
| fi | |
| TARGET_MODES=("$RAW_MODE") | |
| fi | |
| CHECK_INTERVAL=10 | |
| NUM_SAMPLES=1 | |
| NUM_INFERENCE_STEPS=10 | |
| NUM_OUTPUT_FRAMES=500 | |
| AUDIO_NUM_OUTPUT_FRAMES=300 | |
| VIDEO_NUM_OUTPUT_FRAMES=300 | |
| VIDEO_CLIP_MOTION_FPS=50 | |
| DEVICE=cuda:0 | |
| GMR_PY=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/envs/gmr/bin/python | |
| SONIC_RENDER_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/GR00T-WholeBodyControl/gear_sonic | |
| SOURCE_FPS=50 | |
| TARGET_FPS=30 | |
| TEXT_PROMPTS=( | |
| "A person jogs forward at a steady pace." | |
| "A person walks forward calmly." | |
| "A person performs a high jump and lands steadily." | |
| "A person turns around in place." | |
| ) | |
| TEXT_PROMPT_NAMES=( | |
| jog_forward | |
| walk_forward | |
| high_jump | |
| turn_in_place | |
| ) | |
| TEXT_INIT_MOTION_DIRS=( | |
| /cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/Psi0/dataset/seed_all/g1/csv_sonic/230710/jog_ff_loop_360_R_001__A419_M | |
| /cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/Psi0/dataset/seed_all/g1/csv_sonic/230710/walk_ff_loop_315_R_001__A421_M | |
| /cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/Psi0/dataset/seed_all/g1/csv_sonic/230710/high_jump_R_001__A418_M | |
| /cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/Psi0/dataset/seed_all/g1/csv_sonic/230710/idle_turn_360_R_001__A418_M | |
| ) | |
| VIDEO_MOTION_DIRS=( | |
| /cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/GR00T-WholeBodyControl/gear_sonic_deploy/reference/converted/psi0_dance/gvhmr/movies__F-SNBR4jhY_0007 | |
| ) | |
| VIDEO_PATHS=( | |
| /cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/dance_1723_video_psi0/dance_1723_video_10s/movies__F-SNBR4jhY_0007.mp4 | |
| ) | |
| VIDEO_PROMPTS=( | |
| "Generate a motion sequence that matches the provided video." | |
| ) | |
| AUDIO_MOTION_DIRS=( | |
| /cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/GR00T-WholeBodyControl/gear_sonic_deploy/reference/converted/psi0_dance/gvhmr/movies__F-SNBR4jhY_0007 | |
| ) | |
| AUDIO_PATHS=( | |
| /cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/dance_1723_video_psi0/dance_1723_video_10s/movies__F-SNBR4jhY_0007.mp4 | |
| ) | |
| AUDIO_PROMPTS=( | |
| "Generate a dance sequence that matches the provided music." | |
| ) | |
| AUDIO_COMMAND_MOTION_DIRS=( | |
| /cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/Psi0/dataset/seed_all/g1/csv_sonic/230710/walk_ff_loop_315_R_001__A421_M | |
| ) | |
| AUDIO_COMMAND_PATHS=( | |
| /cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/qwen3-tts/seed_audio/003771/female_01_zh.wav | |
| ) | |
| AUDIO_COMMAND_PROMPTS=( | |
| "Generate a motion that follows the spoken instruction in the audio." | |
| ) | |
| RUN_PARENT_DIR=$(dirname "$RUN_DIR") | |
| RUN_NAME=$(basename "$RUN_DIR") | |
| RUN_TAG=$(echo "$RUN_NAME" | awk -F. '{print $NF}') | |
| if [ ${#TARGET_MODES[@]} -gt 1 ]; then | |
| OUTPUT_ROOT=${PSI_AUTO_OUTPUT_DIR:-"$RUN_PARENT_DIR/$RUN_TAG.auto_infer_all"} | |
| else | |
| OUTPUT_ROOT=${PSI_AUTO_OUTPUT_DIR:-"$RUN_PARENT_DIR/$RUN_TAG.auto_infer.${TARGET_MODES[0]}"} | |
| fi | |
| output_dir_for_mode() { | |
| local mode=$1 | |
| if [ ${#TARGET_MODES[@]} -gt 1 ]; then | |
| echo "$OUTPUT_ROOT/$(canonical_mode "$mode")" | |
| else | |
| echo "$OUTPUT_ROOT" | |
| fi | |
| } | |
| expected_samples_for_mode() { | |
| case "$1" in | |
| text|pure_text) | |
| echo "${#TEXT_PROMPTS[@]}" | |
| ;; | |
| audio|audio_text|audio_only|audio_pure) | |
| echo "${#AUDIO_MOTION_DIRS[@]}" | |
| ;; | |
| audio_command|audio_command_text|audio_command_only|command_audio) | |
| echo "${#AUDIO_COMMAND_MOTION_DIRS[@]}" | |
| ;; | |
| video_clip|video_clip_text|video_clip_only|video|video_text|video_only) | |
| echo "${#VIDEO_MOTION_DIRS[@]}" | |
| ;; | |
| *) | |
| echo 0 | |
| ;; | |
| esac | |
| } | |
| expected_motion_frames_for_mode() { | |
| case "$1" in | |
| text|pure_text) | |
| echo "$NUM_OUTPUT_FRAMES" | |
| ;; | |
| audio|audio_text|audio_only|audio_pure|audio_command|audio_command_text|audio_command_only|command_audio) | |
| echo "$AUDIO_NUM_OUTPUT_FRAMES" | |
| ;; | |
| video_clip|video_clip_text|video_clip_only|video|video_text|video_only) | |
| echo "$VIDEO_NUM_OUTPUT_FRAMES" | |
| ;; | |
| *) | |
| echo "$NUM_OUTPUT_FRAMES" | |
| ;; | |
| esac | |
| } | |
| min_render_frames_for_mode() { | |
| local mode=$1 | |
| local motion_frames | |
| motion_frames=$(expected_motion_frames_for_mode "$mode") | |
| python -c 'import sys | |
| motion_frames = int(sys.argv[1]) | |
| source_fps = float(sys.argv[2]) | |
| target_fps = float(sys.argv[3]) | |
| print(int(round((max(motion_frames, 1) - 1) / source_fps * target_fps)) + 1)' "$motion_frames" "$SOURCE_FPS" "$TARGET_FPS" | |
| } | |
| count_valid_render_mp4() { | |
| local step_dir=$1 | |
| local min_render_frames=$2 | |
| local valid_render_mp4_count=0 | |
| local mp4_path | |
| local mp4_frames | |
| while IFS= read -r mp4_path; do | |
| mp4_frames=$(ffprobe -v error -select_streams v:0 -show_entries stream=nb_frames -of default=noprint_wrappers=1:nokey=1 "$mp4_path" 2>/dev/null | head -n1) | |
| if [ -n "$mp4_frames" ] && [ "$mp4_frames" -ge "$min_render_frames" ] 2>/dev/null; then | |
| valid_render_mp4_count=$((valid_render_mp4_count + 1)) | |
| fi | |
| done < <(find "$step_dir" -name 'reference_motion.mp4' -print 2>/dev/null) | |
| echo "$valid_render_mp4_count" | |
| } | |
| run_infer_for_mode() { | |
| local mode=$1 | |
| local ckpt_name=$2 | |
| local ckpt_step=$3 | |
| local step_dir=$4 | |
| local mode_key | |
| local sample_i | |
| local PROMPT | |
| local PROMPT_NAME | |
| local INIT_MOTION_DIR | |
| local MOTION_DIR | |
| local AUDIO_PATH | |
| local VIDEO_PATH | |
| local AUDIO_CONFIG_KEY | |
| local AUDIO_MODE_NAME | |
| local infer_failed=0 | |
| mode_key=$(canonical_mode "$mode") | |
| case "$mode" in | |
| text|pure_text) | |
| for sample_i in "${!TEXT_PROMPTS[@]}"; do | |
| PROMPT=${TEXT_PROMPTS[$sample_i]} | |
| PROMPT_NAME=${TEXT_PROMPT_NAMES[$sample_i]} | |
| INIT_MOTION_DIR=${TEXT_INIT_MOTION_DIRS[$sample_i]} | |
| echo " [推理] 开始 $mode infer sample=$sample_i" | |
| echo " prompt: $PROMPT" | |
| echo " init : $INIT_MOTION_DIR" | |
| "$INFER_PY" scripts/infer/seed_text_to_motion.py \ | |
| --run-dir "$RUN_DIR" \ | |
| --ckpt-step "$ckpt_step" \ | |
| --output-dir "$step_dir" \ | |
| --prompt "$PROMPT" \ | |
| --prompt-motion-name "$PROMPT_NAME" \ | |
| --init-motion-dir "$INIT_MOTION_DIR" \ | |
| --num-inference-steps "$NUM_INFERENCE_STEPS" \ | |
| --num-output-frames "$NUM_OUTPUT_FRAMES" \ | |
| --force-text-only \ | |
| --device "$DEVICE" | |
| if [ $? -ne 0 ]; then | |
| echo " [错误] $ckpt_name $mode infer失败,sample=$sample_i" | |
| infer_failed=1 | |
| break | |
| fi | |
| done | |
| ;; | |
| audio|audio_text|audio_only|audio_pure|audio_command|audio_command_text|audio_command_only|command_audio) | |
| if [ "$mode_key" = "audio" ]; then | |
| AUDIO_CONFIG_KEY=audio | |
| AUDIO_MODE_NAME=audio | |
| MOTION_DIRS=("${AUDIO_MOTION_DIRS[@]}") | |
| AUDIO_FILES=("${AUDIO_PATHS[@]}") | |
| PROMPTS=("${AUDIO_PROMPTS[@]}") | |
| else | |
| AUDIO_CONFIG_KEY=audio_command | |
| AUDIO_MODE_NAME=audio_command | |
| MOTION_DIRS=("${AUDIO_COMMAND_MOTION_DIRS[@]}") | |
| AUDIO_FILES=("${AUDIO_COMMAND_PATHS[@]}") | |
| PROMPTS=("${AUDIO_COMMAND_PROMPTS[@]}") | |
| fi | |
| for sample_i in "${!MOTION_DIRS[@]}"; do | |
| MOTION_DIR=${MOTION_DIRS[$sample_i]} | |
| AUDIO_PATH=${AUDIO_FILES[$sample_i]} | |
| PROMPT=${PROMPTS[$sample_i]} | |
| echo " [推理] 开始 $mode infer sample=$sample_i" | |
| echo " audio_config: $AUDIO_CONFIG_KEY" | |
| echo " motion : $MOTION_DIR" | |
| echo " audio : $AUDIO_PATH" | |
| "$INFER_PY" scripts/infer/seed_audio_to_motion.py \ | |
| --run-dir "$RUN_DIR" \ | |
| --ckpt-step "$ckpt_step" \ | |
| --output-dir "$step_dir" \ | |
| --motion-dir "$MOTION_DIR" \ | |
| --video-path "$AUDIO_PATH" \ | |
| --audio-config-key "$AUDIO_CONFIG_KEY" \ | |
| --prompt "$PROMPT" \ | |
| --num-inference-steps "$NUM_INFERENCE_STEPS" \ | |
| --num-output-frames "$AUDIO_NUM_OUTPUT_FRAMES" \ | |
| --device "$DEVICE" | |
| if [ $? -ne 0 ]; then | |
| echo " [错误] $ckpt_name $AUDIO_MODE_NAME infer失败,sample=$sample_i" | |
| infer_failed=1 | |
| break | |
| fi | |
| done | |
| ;; | |
| video_clip|video_clip_text|video_clip_only|video|video_text|video_only) | |
| for sample_i in "${!VIDEO_MOTION_DIRS[@]}"; do | |
| MOTION_DIR=${VIDEO_MOTION_DIRS[$sample_i]} | |
| VIDEO_PATH=${VIDEO_PATHS[$sample_i]} | |
| PROMPT=${VIDEO_PROMPTS[$sample_i]} | |
| VIDEO_ARGS=(--video-config-key video) | |
| if [ "$mode_key" = "video_clip" ]; then | |
| VIDEO_ARGS=(--video-config-key video_clip --video-clip --video-clip-motion-fps "$VIDEO_CLIP_MOTION_FPS") | |
| fi | |
| if [ "$mode" = "video_only" ] || [ "$mode" = "video_clip_only" ]; then | |
| PROMPT="" | |
| fi | |
| echo " [推理] 开始 $mode infer sample=$sample_i" | |
| echo " motion: $MOTION_DIR" | |
| echo " video : $VIDEO_PATH" | |
| "$INFER_PY" scripts/infer/seed_video_to_motion.py \ | |
| --run-dir "$RUN_DIR" \ | |
| --ckpt-step "$ckpt_step" \ | |
| --output-dir "$step_dir" \ | |
| --motion-dir "$MOTION_DIR" \ | |
| --video-path "$VIDEO_PATH" \ | |
| --prompt "$PROMPT" \ | |
| --num-inference-steps "$NUM_INFERENCE_STEPS" \ | |
| --num-output-frames "$VIDEO_NUM_OUTPUT_FRAMES" \ | |
| "${VIDEO_ARGS[@]}" \ | |
| --device "$DEVICE" | |
| if [ $? -ne 0 ]; then | |
| echo " [错误] $ckpt_name $mode infer失败,sample=$sample_i" | |
| infer_failed=1 | |
| break | |
| fi | |
| done | |
| ;; | |
| *) | |
| echo " [错误] unknown mode: $mode" | |
| infer_failed=1 | |
| ;; | |
| esac | |
| if [ "$infer_failed" -ne 0 ]; then | |
| return 1 | |
| fi | |
| return 0 | |
| } | |
| run_render_for_mode() { | |
| local mode=$1 | |
| local ckpt_name=$2 | |
| local step_dir=$3 | |
| local expected_samples=$4 | |
| local min_render_frames=$5 | |
| local valid_render_mp4_count | |
| echo " [渲染] 开始 $mode render" | |
| ( | |
| cd "$SONIC_RENDER_ROOT" | |
| if [ -n "$XVFB_RUN" ]; then | |
| env -u PYTHONPATH \ | |
| PYOPENGL_PLATFORM=egl \ | |
| MUJOCO_GL=egl \ | |
| QT_X11_NO_MITSHM=1 \ | |
| SONIC_RENDER_HEADLESS_CAMERA=1 \ | |
| SONIC_RENDER_LOCAL_TMP_OUTPUT=1 \ | |
| SONIC_RENDER_TMPDIR=/tmp \ | |
| "$XVFB_RUN" -a -s "-screen 0 1280x1024x24 -extension MIT-SHM" \ | |
| "$GMR_PY" scripts/batch_render_reference_motion_mujoco.py \ | |
| --motion-dir "$step_dir" \ | |
| --source-fps "$SOURCE_FPS" \ | |
| --target-fps "$TARGET_FPS" \ | |
| --dof-order isaaclab | |
| else | |
| echo " [渲染] xvfb-run not found; use headless EGL camera" | |
| env -u PYTHONPATH \ | |
| PYOPENGL_PLATFORM=egl \ | |
| MUJOCO_GL=egl \ | |
| QT_X11_NO_MITSHM=1 \ | |
| SONIC_RENDER_HEADLESS_CAMERA=1 \ | |
| SONIC_RENDER_LOCAL_TMP_OUTPUT=1 \ | |
| SONIC_RENDER_TMPDIR=/tmp \ | |
| "$GMR_PY" scripts/batch_render_reference_motion_mujoco.py \ | |
| --motion-dir "$step_dir" \ | |
| --source-fps "$SOURCE_FPS" \ | |
| --target-fps "$TARGET_FPS" \ | |
| --dof-order isaaclab | |
| fi | |
| ) | |
| if [ $? -ne 0 ]; then | |
| valid_render_mp4_count=$(count_valid_render_mp4 "$step_dir" "$min_render_frames") | |
| if [ "$valid_render_mp4_count" -ge "$expected_samples" ]; then | |
| echo " [警告] $mode render返回非0,但已生成 ${valid_render_mp4_count}/${expected_samples} 个有效reference_motion.mp4,视为完成" | |
| else | |
| echo " [错误] $ckpt_name $mode render失败,当前有效reference_motion.mp4数量: $valid_render_mp4_count/${expected_samples}" | |
| return 1 | |
| fi | |
| else | |
| echo " [完成] $ckpt_name $mode render成功" | |
| fi | |
| return 0 | |
| } | |
| process_mode_for_ckpt() { | |
| local mode=$1 | |
| local ckpt_name=$2 | |
| local ckpt_step=$3 | |
| local output_dir | |
| local step_dir | |
| local expected_samples | |
| local infer_csv_count | |
| local min_render_frames | |
| local valid_render_mp4_count | |
| output_dir=$(output_dir_for_mode "$mode") | |
| step_dir="$output_dir/step_$ckpt_step" | |
| expected_samples=$(expected_samples_for_mode "$mode") | |
| min_render_frames=$(min_render_frames_for_mode "$mode") | |
| mkdir -p "$step_dir" | |
| infer_csv_count=$(find "$step_dir" -name 'body_pos.csv' -print 2>/dev/null | wc -l | tr -d ' ') | |
| valid_render_mp4_count=$(count_valid_render_mp4 "$step_dir" "$min_render_frames") | |
| echo " [模式] $mode" | |
| echo " 输出目录: $step_dir" | |
| echo " infer: ${infer_csv_count}/${expected_samples}" | |
| echo " render: ${valid_render_mp4_count}/${expected_samples}" | |
| if [ "$infer_csv_count" -lt "$expected_samples" ]; then | |
| if ! run_infer_for_mode "$mode" "$ckpt_name" "$ckpt_step" "$step_dir"; then | |
| return 1 | |
| fi | |
| infer_csv_count=$(find "$step_dir" -name 'body_pos.csv' -print 2>/dev/null | wc -l | tr -d ' ') | |
| echo " [完成] $ckpt_name $mode infer成功,当前body_pos.csv数量: $infer_csv_count" | |
| else | |
| echo " [跳过] $mode infer已完成: ${infer_csv_count}/${expected_samples} 个body_pos.csv" | |
| fi | |
| valid_render_mp4_count=$(count_valid_render_mp4 "$step_dir" "$min_render_frames") | |
| if [ "$valid_render_mp4_count" -lt "$expected_samples" ]; then | |
| if ! run_render_for_mode "$mode" "$ckpt_name" "$step_dir" "$expected_samples" "$min_render_frames"; then | |
| return 1 | |
| fi | |
| else | |
| echo " [跳过] $mode render已完成: ${valid_render_mp4_count}/${expected_samples} 个有效reference_motion.mp4" | |
| fi | |
| return 0 | |
| } | |
| if [ -x /usr/bin/xvfb-run ]; then | |
| XVFB_RUN=/usr/bin/xvfb-run | |
| elif [ -x /bin/xvfb-run ]; then | |
| XVFB_RUN=/bin/xvfb-run | |
| elif command -v xvfb-run >/dev/null 2>&1; then | |
| XVFB_RUN=$(command -v xvfb-run) | |
| else | |
| XVFB_RUN= | |
| fi | |
| export PYOPENGL_PLATFORM=egl | |
| export MUJOCO_GL=egl | |
| export QT_X11_NO_MITSHM=1 | |
| echo "==========================================" | |
| echo "Auto Infer + Render Monitor" | |
| echo "==========================================" | |
| echo "RUN_DIR: $RUN_DIR" | |
| echo "OUTPUT_ROOT: $OUTPUT_ROOT" | |
| echo "RAW_MODE: $RAW_MODE" | |
| echo "MODE: $MODE" | |
| echo "SUPPORTED_MODES: ${SUPPORTED_MODES[*]}" | |
| echo "TARGET_MODES: ${TARGET_MODES[*]}" | |
| echo "CHECK_INTERVAL: ${CHECK_INTERVAL}s" | |
| echo "NUM_OUTPUT_FRAMES: $NUM_OUTPUT_FRAMES" | |
| echo "AUDIO_NUM_OUTPUT_FRAMES: $AUDIO_NUM_OUTPUT_FRAMES" | |
| echo "VIDEO_NUM_OUTPUT_FRAMES: $VIDEO_NUM_OUTPUT_FRAMES" | |
| echo "NUM_INFERENCE_STEPS: $NUM_INFERENCE_STEPS" | |
| echo "XVFB_RUN: ${XVFB_RUN:-<not found, use headless EGL camera>}" | |
| echo "NOTE: default MODE=all. Set PSI_AUTO_INFER_MODE to run a single modality." | |
| echo "NOTE: single-mode values: text / audio / audio_command / video_clip" | |
| echo "==========================================" | |
| mkdir -p "$OUTPUT_ROOT" | |
| for mode in "${TARGET_MODES[@]}"; do | |
| mode_output_dir=$(output_dir_for_mode "$mode") | |
| mkdir -p "$mode_output_dir" | |
| echo "OUTPUT_DIR[$mode]: $mode_output_dir" | |
| done | |
| while true; do | |
| echo "" | |
| echo "[$(date '+%Y-%m-%d %H:%M:%S')] 扫描新的checkpoint..." | |
| pending_ckpts=() | |
| checkpoint_dirs=() | |
| for ckpt_dir in "$RUN_DIR"/checkpoints/ckpt_*; do | |
| [ -d "$ckpt_dir" ] || continue | |
| checkpoint_dirs+=("$ckpt_dir") | |
| done | |
| if [ ${#checkpoint_dirs[@]} -eq 0 ] && [ -d "$RUN_DIR/checkpoints/best_ckpt" ]; then | |
| checkpoint_dirs+=("$RUN_DIR/checkpoints/best_ckpt") | |
| fi | |
| for ckpt_dir in "${checkpoint_dirs[@]}"; do | |
| ckpt_name=$(basename "$ckpt_dir") | |
| if [ "$ckpt_name" = "best_ckpt" ]; then | |
| ckpt_step=best | |
| sort_step=999999999999 | |
| elif [[ "$ckpt_name" =~ ckpt_([0-9]+) ]]; then | |
| ckpt_step="${BASH_REMATCH[1]}" | |
| sort_step="$ckpt_step" | |
| else | |
| echo " [警告] 无法从 $ckpt_name 提取step数,跳过" | |
| continue | |
| fi | |
| ckpt_pending_modes=() | |
| for scan_mode in "${TARGET_MODES[@]}"; do | |
| scan_step_dir="$(output_dir_for_mode "$scan_mode")/step_$ckpt_step" | |
| scan_expected_samples=$(expected_samples_for_mode "$scan_mode") | |
| scan_min_render_frames=$(min_render_frames_for_mode "$scan_mode") | |
| scan_valid_render_mp4_count=$(count_valid_render_mp4 "$scan_step_dir" "$scan_min_render_frames") | |
| if [ "$scan_valid_render_mp4_count" -lt "$scan_expected_samples" ]; then | |
| ckpt_pending_modes+=("$scan_mode") | |
| fi | |
| done | |
| if [ ${#ckpt_pending_modes[@]} -eq 0 ]; then | |
| echo " [跳过] $ckpt_name (所有目标模态都已完成)" | |
| continue | |
| fi | |
| pending_modes_csv=$(IFS=,; echo "${ckpt_pending_modes[*]}") | |
| pending_ckpts+=("${sort_step}|${ckpt_step}|${ckpt_dir}|${pending_modes_csv}") | |
| done | |
| if [ ${#pending_ckpts[@]} -eq 0 ]; then | |
| echo " [无] 没有待处理的checkpoint" | |
| echo "[$(date '+%Y-%m-%d %H:%M:%S')] 等待 ${CHECK_INTERVAL} 秒后再次检查..." | |
| sleep "$CHECK_INTERVAL" | |
| continue | |
| fi | |
| best_item=$(printf '%s\n' "${pending_ckpts[@]}" | sort -t'|' -k1,1nr | head -n1) | |
| IFS='|' read -r sort_step ckpt_step ckpt_dir pending_modes_csv <<< "$best_item" | |
| ckpt_name=$(basename "$ckpt_dir") | |
| IFS=',' read -r -a CKPT_PENDING_MODES <<< "$pending_modes_csv" | |
| echo " [发现] 待处理checkpoint共 ${#pending_ckpts[@]} 个,选择step最大的:" | |
| echo " $ckpt_name" | |
| echo " [步骤] step: $ckpt_step" | |
| echo " [模式] 待处理: ${CKPT_PENDING_MODES[*]}" | |
| mode_failed=0 | |
| for mode in "${CKPT_PENDING_MODES[@]}"; do | |
| if ! process_mode_for_ckpt "$mode" "$ckpt_name" "$ckpt_step"; then | |
| mode_failed=1 | |
| break | |
| fi | |
| done | |
| if [ "$mode_failed" -ne 0 ]; then | |
| echo "[$(date '+%Y-%m-%d %H:%M:%S')] 等待 ${CHECK_INTERVAL} 秒后再次检查..." | |
| sleep "$CHECK_INTERVAL" | |
| continue | |
| fi | |
| echo "[$(date '+%Y-%m-%d %H:%M:%S')] 等待 ${CHECK_INTERVAL} 秒后再次检查..." | |
| sleep "$CHECK_INTERVAL" | |
| done | |