#!/bin/bash REPO_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/Psi0-yth-v21 DEFAULT_RUN_DIR=$REPO_ROOT/runs/posttrain_v1_trimodal/posttrain_actionhead_M.sg.flow1000.cwml.lr1.0e-04.b256.gpus8.2606241549 cd "$REPO_ROOT" export PYTHONPATH="$REPO_ROOT/src" source /cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/Psi0-yth-v15/.venv-psi/bin/activate [ -f .env ] && source .env export PYTHONPATH="$REPO_ROOT/src" export CUDA_VISIBLE_DEVICES=${CUDA_VISIBLE_DEVICES:-0} export PSI_INFER_FLOAT32=1 pkill -f monitor.py || true canonical_mode() { case "$1" in all|multi|full) echo all ;; text|pure_text) echo text ;; audio|audio_text|audio_only|audio_pure) echo audio ;; audio_command|audio_command_text|audio_command_only|command_audio) echo audio_command ;; video_clip|video_clip_text|video_clip_only) echo video_clip ;; video|video_text|video_only) echo video ;; *) echo "$1" ;; esac } RAW_MODE=${PSI_AUTO_INFER_MODE:-all} MODE=$(canonical_mode "$RAW_MODE") MODE_SEQUENCE=(text audio audio_command video_clip) # MODE_SEQUENCE=(audio_command) RUNS_ROOT=${RUNS_ROOT:-$REPO_ROOT/runs/posttrain_v1_trimodal} INFER_PY=${INFER_PY:-/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/Psi0-yth-v15/.venv-psi/bin/python} # RUN_DIR=${RUN_DIR:-auto} RUN_DIR=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/Psi0-yth-v21/runs/posttrain_v1_trimodal/posttrain.sg.flow1000.cwml.lr1.0e-04.b512.gpus8.2607051648 if [ $# -ge 1 ]; then RUN_DIR=$1 elif [ "$RUN_DIR" = "auto" ] || [ -z "$RUN_DIR" ]; then AUTO_RUN_DIR=$("$INFER_PY" - "$MODE" "$RUNS_ROOT" <<'PY' import json import re import sys from pathlib import Path mode = sys.argv[1] runs_root = Path(sys.argv[2]) def ratio(data, key): try: return float(data.get(key, 0.0)) except Exception: return 0.0 def supported_modes(run_config): data = run_config.get("data", {}) modes = [] if ratio(data, "text_ratio") > 0 or "root_dir" in data: modes.append("text") if ratio(data, "audio_ratio") > 0 or ratio(data, "audio_only_ratio") > 0: modes.append("audio") if ratio(data, "audio_command_ratio") > 0 or ratio(data, "audio_command_only_ratio") > 0: modes.append("audio_command") if ratio(data, "video_clip_text_ratio") > 0 or ratio(data, "video_clip_only_ratio") > 0: modes.append("video_clip") if ratio(data, "video_ratio") > 0 or ratio(data, "video_only_ratio") > 0: modes.append("video") return modes def run_sort_key(path): match = re.search(r"(\d{10})$", path.name) if match: return int(match.group(1)) return int(path.stat().st_mtime) candidates = [] for path in runs_root.glob("posttrain*.gpus8.*"): config_path = path / "run_config.json" if not config_path.exists(): continue try: run_config = json.loads(config_path.read_text()) except Exception: continue modes = supported_modes(run_config) if mode == "all": score = len([m for m in ("text", "audio", "audio_command", "video_clip", "video") if m in modes]) else: score = 1 if mode in modes else 0 if score > 0: candidates.append((score, run_sort_key(path), path)) if candidates: candidates.sort() print(str(candidates[-1][2])) PY ) if [ -n "$AUTO_RUN_DIR" ]; then RUN_DIR=$AUTO_RUN_DIR else RUN_DIR=$DEFAULT_RUN_DIR fi fi if [ -z "$RUN_DIR" ] || [ ! -f "$RUN_DIR/run_config.json" ]; then echo "ERROR: cannot resolve a valid RUN_DIR for MODE=$MODE" echo " Pass it explicitly: bash infer_render_auto.sh /path/to/run_dir" exit 1 fi SUPPORTED_MODES_CSV=$("$INFER_PY" - "$RUN_DIR" <<'PY' import json import sys from pathlib import Path run_dir = Path(sys.argv[1]) config = json.loads((run_dir / "run_config.json").read_text()) data = config.get("data", {}) def ratio(key): try: return float(data.get(key, 0.0)) except Exception: return 0.0 supported = [] if ratio("text_ratio") > 0 or "root_dir" in data: supported.append("text") if ratio("audio_ratio") > 0 or ratio("audio_only_ratio") > 0: supported.append("audio") if ratio("audio_command_ratio") > 0 or ratio("audio_command_only_ratio") > 0: supported.append("audio_command") if ratio("video_clip_text_ratio") > 0 or ratio("video_clip_only_ratio") > 0: supported.append("video_clip") if ratio("video_ratio") > 0 or ratio("video_only_ratio") > 0: supported.append("video") print(",".join(supported)) PY ) if [ -z "$SUPPORTED_MODES_CSV" ]; then echo "ERROR: RUN_DIR does not expose any supported inference modes: $RUN_DIR" exit 1 fi IFS=',' read -r -a SUPPORTED_MODES <<< "$SUPPORTED_MODES_CSV" mode_supported() { local wanted local supported wanted=$(canonical_mode "$1") for supported in "${SUPPORTED_MODES[@]}"; do if [ "$supported" = "$wanted" ]; then return 0 fi done return 1 } TARGET_MODES=() if [ "$MODE" = "all" ]; then if mode_supported text; then TARGET_MODES+=(text) fi if mode_supported audio; then TARGET_MODES+=(audio) fi if mode_supported audio_command; then TARGET_MODES+=(audio_command) fi if mode_supported video_clip; then TARGET_MODES+=(video_clip) elif mode_supported video; then TARGET_MODES+=(video) fi if [ ${#TARGET_MODES[@]} -eq 0 ]; then echo "ERROR: MODE=all but RUN_DIR has no supported target modes: $RUN_DIR" exit 1 fi else if ! mode_supported "$RAW_MODE"; then echo "ERROR: RUN_DIR does not look compatible with MODE=$RAW_MODE: $RUN_DIR" echo " supported modes: ${SUPPORTED_MODES[*]}" exit 1 fi TARGET_MODES=("$RAW_MODE") fi CHECK_INTERVAL=10 NUM_SAMPLES=1 NUM_INFERENCE_STEPS=10 NUM_OUTPUT_FRAMES=500 AUDIO_NUM_OUTPUT_FRAMES=300 VIDEO_NUM_OUTPUT_FRAMES=300 VIDEO_CLIP_MOTION_FPS=50 DEVICE=cuda:0 GMR_PY=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/envs/gmr/bin/python SONIC_RENDER_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/GR00T-WholeBodyControl/gear_sonic SOURCE_FPS=50 TARGET_FPS=30 TEXT_PROMPTS=( "A person jogs forward at a steady pace." "A person walks forward calmly." "A person performs a high jump and lands steadily." "A person turns around in place." ) TEXT_PROMPT_NAMES=( jog_forward walk_forward high_jump turn_in_place ) TEXT_INIT_MOTION_DIRS=( /cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/Psi0/dataset/seed_all/g1/csv_sonic/230710/jog_ff_loop_360_R_001__A419_M /cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/Psi0/dataset/seed_all/g1/csv_sonic/230710/walk_ff_loop_315_R_001__A421_M /cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/Psi0/dataset/seed_all/g1/csv_sonic/230710/high_jump_R_001__A418_M /cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/Psi0/dataset/seed_all/g1/csv_sonic/230710/idle_turn_360_R_001__A418_M ) VIDEO_MOTION_DIRS=( /cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/GR00T-WholeBodyControl/gear_sonic_deploy/reference/converted/psi0_dance/gvhmr/movies__F-SNBR4jhY_0007 ) VIDEO_PATHS=( /cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/dance_1723_video_psi0/dance_1723_video_10s/movies__F-SNBR4jhY_0007.mp4 ) VIDEO_PROMPTS=( "Generate a motion sequence that matches the provided video." ) AUDIO_MOTION_DIRS=( /cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/GR00T-WholeBodyControl/gear_sonic_deploy/reference/converted/psi0_dance/gvhmr/movies__F-SNBR4jhY_0007 ) AUDIO_PATHS=( /cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/dance_1723_video_psi0/dance_1723_video_10s/movies__F-SNBR4jhY_0007.mp4 ) AUDIO_PROMPTS=( "Generate a dance sequence that matches the provided music." ) AUDIO_COMMAND_MOTION_DIRS=( /cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/Psi0/dataset/seed_all/g1/csv_sonic/230710/walk_ff_loop_315_R_001__A421_M ) AUDIO_COMMAND_PATHS=( /cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/qwen3-tts/seed_audio/003771/female_01_zh.wav ) AUDIO_COMMAND_PROMPTS=( "Generate a motion that follows the spoken instruction in the audio." ) RUN_PARENT_DIR=$(dirname "$RUN_DIR") RUN_NAME=$(basename "$RUN_DIR") RUN_TAG=$(echo "$RUN_NAME" | awk -F. '{print $NF}') if [ ${#TARGET_MODES[@]} -gt 1 ]; then OUTPUT_ROOT=${PSI_AUTO_OUTPUT_DIR:-"$RUN_PARENT_DIR/$RUN_TAG.auto_infer_all"} else OUTPUT_ROOT=${PSI_AUTO_OUTPUT_DIR:-"$RUN_PARENT_DIR/$RUN_TAG.auto_infer.${TARGET_MODES[0]}"} fi output_dir_for_mode() { local mode=$1 if [ ${#TARGET_MODES[@]} -gt 1 ]; then echo "$OUTPUT_ROOT/$(canonical_mode "$mode")" else echo "$OUTPUT_ROOT" fi } expected_samples_for_mode() { case "$1" in text|pure_text) echo "${#TEXT_PROMPTS[@]}" ;; audio|audio_text|audio_only|audio_pure) echo "${#AUDIO_MOTION_DIRS[@]}" ;; audio_command|audio_command_text|audio_command_only|command_audio) echo "${#AUDIO_COMMAND_MOTION_DIRS[@]}" ;; video_clip|video_clip_text|video_clip_only|video|video_text|video_only) echo "${#VIDEO_MOTION_DIRS[@]}" ;; *) echo 0 ;; esac } expected_motion_frames_for_mode() { case "$1" in text|pure_text) echo "$NUM_OUTPUT_FRAMES" ;; audio|audio_text|audio_only|audio_pure|audio_command|audio_command_text|audio_command_only|command_audio) echo "$AUDIO_NUM_OUTPUT_FRAMES" ;; video_clip|video_clip_text|video_clip_only|video|video_text|video_only) echo "$VIDEO_NUM_OUTPUT_FRAMES" ;; *) echo "$NUM_OUTPUT_FRAMES" ;; esac } min_render_frames_for_mode() { local mode=$1 local motion_frames motion_frames=$(expected_motion_frames_for_mode "$mode") python -c 'import sys motion_frames = int(sys.argv[1]) source_fps = float(sys.argv[2]) target_fps = float(sys.argv[3]) print(int(round((max(motion_frames, 1) - 1) / source_fps * target_fps)) + 1)' "$motion_frames" "$SOURCE_FPS" "$TARGET_FPS" } count_valid_render_mp4() { local step_dir=$1 local min_render_frames=$2 local valid_render_mp4_count=0 local mp4_path local mp4_frames while IFS= read -r mp4_path; do mp4_frames=$(ffprobe -v error -select_streams v:0 -show_entries stream=nb_frames -of default=noprint_wrappers=1:nokey=1 "$mp4_path" 2>/dev/null | head -n1) if [ -n "$mp4_frames" ] && [ "$mp4_frames" -ge "$min_render_frames" ] 2>/dev/null; then valid_render_mp4_count=$((valid_render_mp4_count + 1)) fi done < <(find "$step_dir" -name 'reference_motion.mp4' -print 2>/dev/null) echo "$valid_render_mp4_count" } run_infer_for_mode() { local mode=$1 local ckpt_name=$2 local ckpt_step=$3 local step_dir=$4 local mode_key local sample_i local PROMPT local PROMPT_NAME local INIT_MOTION_DIR local MOTION_DIR local AUDIO_PATH local VIDEO_PATH local AUDIO_CONFIG_KEY local AUDIO_MODE_NAME local infer_failed=0 mode_key=$(canonical_mode "$mode") case "$mode" in text|pure_text) for sample_i in "${!TEXT_PROMPTS[@]}"; do PROMPT=${TEXT_PROMPTS[$sample_i]} PROMPT_NAME=${TEXT_PROMPT_NAMES[$sample_i]} INIT_MOTION_DIR=${TEXT_INIT_MOTION_DIRS[$sample_i]} echo " [推理] 开始 $mode infer sample=$sample_i" echo " prompt: $PROMPT" echo " init : $INIT_MOTION_DIR" "$INFER_PY" scripts/infer/seed_text_to_motion.py \ --run-dir "$RUN_DIR" \ --ckpt-step "$ckpt_step" \ --output-dir "$step_dir" \ --prompt "$PROMPT" \ --prompt-motion-name "$PROMPT_NAME" \ --init-motion-dir "$INIT_MOTION_DIR" \ --num-inference-steps "$NUM_INFERENCE_STEPS" \ --num-output-frames "$NUM_OUTPUT_FRAMES" \ --force-text-only \ --device "$DEVICE" if [ $? -ne 0 ]; then echo " [错误] $ckpt_name $mode infer失败,sample=$sample_i" infer_failed=1 break fi done ;; audio|audio_text|audio_only|audio_pure|audio_command|audio_command_text|audio_command_only|command_audio) if [ "$mode_key" = "audio" ]; then AUDIO_CONFIG_KEY=audio AUDIO_MODE_NAME=audio MOTION_DIRS=("${AUDIO_MOTION_DIRS[@]}") AUDIO_FILES=("${AUDIO_PATHS[@]}") PROMPTS=("${AUDIO_PROMPTS[@]}") else AUDIO_CONFIG_KEY=audio_command AUDIO_MODE_NAME=audio_command MOTION_DIRS=("${AUDIO_COMMAND_MOTION_DIRS[@]}") AUDIO_FILES=("${AUDIO_COMMAND_PATHS[@]}") PROMPTS=("${AUDIO_COMMAND_PROMPTS[@]}") fi for sample_i in "${!MOTION_DIRS[@]}"; do MOTION_DIR=${MOTION_DIRS[$sample_i]} AUDIO_PATH=${AUDIO_FILES[$sample_i]} PROMPT=${PROMPTS[$sample_i]} echo " [推理] 开始 $mode infer sample=$sample_i" echo " audio_config: $AUDIO_CONFIG_KEY" echo " motion : $MOTION_DIR" echo " audio : $AUDIO_PATH" "$INFER_PY" scripts/infer/seed_audio_to_motion.py \ --run-dir "$RUN_DIR" \ --ckpt-step "$ckpt_step" \ --output-dir "$step_dir" \ --motion-dir "$MOTION_DIR" \ --video-path "$AUDIO_PATH" \ --audio-config-key "$AUDIO_CONFIG_KEY" \ --prompt "$PROMPT" \ --num-inference-steps "$NUM_INFERENCE_STEPS" \ --num-output-frames "$AUDIO_NUM_OUTPUT_FRAMES" \ --device "$DEVICE" if [ $? -ne 0 ]; then echo " [错误] $ckpt_name $AUDIO_MODE_NAME infer失败,sample=$sample_i" infer_failed=1 break fi done ;; video_clip|video_clip_text|video_clip_only|video|video_text|video_only) for sample_i in "${!VIDEO_MOTION_DIRS[@]}"; do MOTION_DIR=${VIDEO_MOTION_DIRS[$sample_i]} VIDEO_PATH=${VIDEO_PATHS[$sample_i]} PROMPT=${VIDEO_PROMPTS[$sample_i]} VIDEO_ARGS=(--video-config-key video) if [ "$mode_key" = "video_clip" ]; then VIDEO_ARGS=(--video-config-key video_clip --video-clip --video-clip-motion-fps "$VIDEO_CLIP_MOTION_FPS") fi if [ "$mode" = "video_only" ] || [ "$mode" = "video_clip_only" ]; then PROMPT="" fi echo " [推理] 开始 $mode infer sample=$sample_i" echo " motion: $MOTION_DIR" echo " video : $VIDEO_PATH" "$INFER_PY" scripts/infer/seed_video_to_motion.py \ --run-dir "$RUN_DIR" \ --ckpt-step "$ckpt_step" \ --output-dir "$step_dir" \ --motion-dir "$MOTION_DIR" \ --video-path "$VIDEO_PATH" \ --prompt "$PROMPT" \ --num-inference-steps "$NUM_INFERENCE_STEPS" \ --num-output-frames "$VIDEO_NUM_OUTPUT_FRAMES" \ "${VIDEO_ARGS[@]}" \ --device "$DEVICE" if [ $? -ne 0 ]; then echo " [错误] $ckpt_name $mode infer失败,sample=$sample_i" infer_failed=1 break fi done ;; *) echo " [错误] unknown mode: $mode" infer_failed=1 ;; esac if [ "$infer_failed" -ne 0 ]; then return 1 fi return 0 } run_render_for_mode() { local mode=$1 local ckpt_name=$2 local step_dir=$3 local expected_samples=$4 local min_render_frames=$5 local valid_render_mp4_count echo " [渲染] 开始 $mode render" ( cd "$SONIC_RENDER_ROOT" if [ -n "$XVFB_RUN" ]; then env -u PYTHONPATH \ PYOPENGL_PLATFORM=egl \ MUJOCO_GL=egl \ QT_X11_NO_MITSHM=1 \ SONIC_RENDER_HEADLESS_CAMERA=1 \ SONIC_RENDER_LOCAL_TMP_OUTPUT=1 \ SONIC_RENDER_TMPDIR=/tmp \ "$XVFB_RUN" -a -s "-screen 0 1280x1024x24 -extension MIT-SHM" \ "$GMR_PY" scripts/batch_render_reference_motion_mujoco.py \ --motion-dir "$step_dir" \ --source-fps "$SOURCE_FPS" \ --target-fps "$TARGET_FPS" \ --dof-order isaaclab else echo " [渲染] xvfb-run not found; use headless EGL camera" env -u PYTHONPATH \ PYOPENGL_PLATFORM=egl \ MUJOCO_GL=egl \ QT_X11_NO_MITSHM=1 \ SONIC_RENDER_HEADLESS_CAMERA=1 \ SONIC_RENDER_LOCAL_TMP_OUTPUT=1 \ SONIC_RENDER_TMPDIR=/tmp \ "$GMR_PY" scripts/batch_render_reference_motion_mujoco.py \ --motion-dir "$step_dir" \ --source-fps "$SOURCE_FPS" \ --target-fps "$TARGET_FPS" \ --dof-order isaaclab fi ) if [ $? -ne 0 ]; then valid_render_mp4_count=$(count_valid_render_mp4 "$step_dir" "$min_render_frames") if [ "$valid_render_mp4_count" -ge "$expected_samples" ]; then echo " [警告] $mode render返回非0,但已生成 ${valid_render_mp4_count}/${expected_samples} 个有效reference_motion.mp4,视为完成" else echo " [错误] $ckpt_name $mode render失败,当前有效reference_motion.mp4数量: $valid_render_mp4_count/${expected_samples}" return 1 fi else echo " [完成] $ckpt_name $mode render成功" fi return 0 } process_mode_for_ckpt() { local mode=$1 local ckpt_name=$2 local ckpt_step=$3 local output_dir local step_dir local expected_samples local infer_csv_count local min_render_frames local valid_render_mp4_count output_dir=$(output_dir_for_mode "$mode") step_dir="$output_dir/step_$ckpt_step" expected_samples=$(expected_samples_for_mode "$mode") min_render_frames=$(min_render_frames_for_mode "$mode") mkdir -p "$step_dir" infer_csv_count=$(find "$step_dir" -name 'body_pos.csv' -print 2>/dev/null | wc -l | tr -d ' ') valid_render_mp4_count=$(count_valid_render_mp4 "$step_dir" "$min_render_frames") echo " [模式] $mode" echo " 输出目录: $step_dir" echo " infer: ${infer_csv_count}/${expected_samples}" echo " render: ${valid_render_mp4_count}/${expected_samples}" if [ "$infer_csv_count" -lt "$expected_samples" ]; then if ! run_infer_for_mode "$mode" "$ckpt_name" "$ckpt_step" "$step_dir"; then return 1 fi infer_csv_count=$(find "$step_dir" -name 'body_pos.csv' -print 2>/dev/null | wc -l | tr -d ' ') echo " [完成] $ckpt_name $mode infer成功,当前body_pos.csv数量: $infer_csv_count" else echo " [跳过] $mode infer已完成: ${infer_csv_count}/${expected_samples} 个body_pos.csv" fi valid_render_mp4_count=$(count_valid_render_mp4 "$step_dir" "$min_render_frames") if [ "$valid_render_mp4_count" -lt "$expected_samples" ]; then if ! run_render_for_mode "$mode" "$ckpt_name" "$step_dir" "$expected_samples" "$min_render_frames"; then return 1 fi else echo " [跳过] $mode render已完成: ${valid_render_mp4_count}/${expected_samples} 个有效reference_motion.mp4" fi return 0 } if [ -x /usr/bin/xvfb-run ]; then XVFB_RUN=/usr/bin/xvfb-run elif [ -x /bin/xvfb-run ]; then XVFB_RUN=/bin/xvfb-run elif command -v xvfb-run >/dev/null 2>&1; then XVFB_RUN=$(command -v xvfb-run) else XVFB_RUN= fi export PYOPENGL_PLATFORM=egl export MUJOCO_GL=egl export QT_X11_NO_MITSHM=1 echo "==========================================" echo "Auto Infer + Render Monitor" echo "==========================================" echo "RUN_DIR: $RUN_DIR" echo "OUTPUT_ROOT: $OUTPUT_ROOT" echo "RAW_MODE: $RAW_MODE" echo "MODE: $MODE" echo "SUPPORTED_MODES: ${SUPPORTED_MODES[*]}" echo "TARGET_MODES: ${TARGET_MODES[*]}" echo "CHECK_INTERVAL: ${CHECK_INTERVAL}s" echo "NUM_OUTPUT_FRAMES: $NUM_OUTPUT_FRAMES" echo "AUDIO_NUM_OUTPUT_FRAMES: $AUDIO_NUM_OUTPUT_FRAMES" echo "VIDEO_NUM_OUTPUT_FRAMES: $VIDEO_NUM_OUTPUT_FRAMES" echo "NUM_INFERENCE_STEPS: $NUM_INFERENCE_STEPS" echo "XVFB_RUN: ${XVFB_RUN:-}" echo "NOTE: default MODE=all. Set PSI_AUTO_INFER_MODE to run a single modality." echo "NOTE: single-mode values: text / audio / audio_command / video_clip" echo "==========================================" mkdir -p "$OUTPUT_ROOT" for mode in "${TARGET_MODES[@]}"; do mode_output_dir=$(output_dir_for_mode "$mode") mkdir -p "$mode_output_dir" echo "OUTPUT_DIR[$mode]: $mode_output_dir" done while true; do echo "" echo "[$(date '+%Y-%m-%d %H:%M:%S')] 扫描新的checkpoint..." pending_ckpts=() checkpoint_dirs=() for ckpt_dir in "$RUN_DIR"/checkpoints/ckpt_*; do [ -d "$ckpt_dir" ] || continue checkpoint_dirs+=("$ckpt_dir") done if [ ${#checkpoint_dirs[@]} -eq 0 ] && [ -d "$RUN_DIR/checkpoints/best_ckpt" ]; then checkpoint_dirs+=("$RUN_DIR/checkpoints/best_ckpt") fi for ckpt_dir in "${checkpoint_dirs[@]}"; do ckpt_name=$(basename "$ckpt_dir") if [ "$ckpt_name" = "best_ckpt" ]; then ckpt_step=best sort_step=999999999999 elif [[ "$ckpt_name" =~ ckpt_([0-9]+) ]]; then ckpt_step="${BASH_REMATCH[1]}" sort_step="$ckpt_step" else echo " [警告] 无法从 $ckpt_name 提取step数,跳过" continue fi ckpt_pending_modes=() for scan_mode in "${TARGET_MODES[@]}"; do scan_step_dir="$(output_dir_for_mode "$scan_mode")/step_$ckpt_step" scan_expected_samples=$(expected_samples_for_mode "$scan_mode") scan_min_render_frames=$(min_render_frames_for_mode "$scan_mode") scan_valid_render_mp4_count=$(count_valid_render_mp4 "$scan_step_dir" "$scan_min_render_frames") if [ "$scan_valid_render_mp4_count" -lt "$scan_expected_samples" ]; then ckpt_pending_modes+=("$scan_mode") fi done if [ ${#ckpt_pending_modes[@]} -eq 0 ]; then echo " [跳过] $ckpt_name (所有目标模态都已完成)" continue fi pending_modes_csv=$(IFS=,; echo "${ckpt_pending_modes[*]}") pending_ckpts+=("${sort_step}|${ckpt_step}|${ckpt_dir}|${pending_modes_csv}") done if [ ${#pending_ckpts[@]} -eq 0 ]; then echo " [无] 没有待处理的checkpoint" echo "[$(date '+%Y-%m-%d %H:%M:%S')] 等待 ${CHECK_INTERVAL} 秒后再次检查..." sleep "$CHECK_INTERVAL" continue fi best_item=$(printf '%s\n' "${pending_ckpts[@]}" | sort -t'|' -k1,1nr | head -n1) IFS='|' read -r sort_step ckpt_step ckpt_dir pending_modes_csv <<< "$best_item" ckpt_name=$(basename "$ckpt_dir") IFS=',' read -r -a CKPT_PENDING_MODES <<< "$pending_modes_csv" echo " [发现] 待处理checkpoint共 ${#pending_ckpts[@]} 个,选择step最大的:" echo " $ckpt_name" echo " [步骤] step: $ckpt_step" echo " [模式] 待处理: ${CKPT_PENDING_MODES[*]}" mode_failed=0 for mode in "${CKPT_PENDING_MODES[@]}"; do if ! process_mode_for_ckpt "$mode" "$ckpt_name" "$ckpt_step"; then mode_failed=1 break fi done if [ "$mode_failed" -ne 0 ]; then echo "[$(date '+%Y-%m-%d %H:%M:%S')] 等待 ${CHECK_INTERVAL} 秒后再次检查..." sleep "$CHECK_INTERVAL" continue fi echo "[$(date '+%Y-%m-%d %H:%M:%S')] 等待 ${CHECK_INTERVAL} 秒后再次检查..." sleep "$CHECK_INTERVAL" done