code-Psi0-yth-v21 / infer_render_auto.sh
Tinghong-Ye22's picture
Add files using upload-large-folder tool
d8aa1ac verified
Raw History Blame Contribute Delete
25.6 kB
#!/bin/bash
REPO_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/Psi0-yth-v21
DEFAULT_RUN_DIR=$REPO_ROOT/runs/posttrain_v1_trimodal/posttrain_actionhead_M.sg.flow1000.cwml.lr1.0e-04.b256.gpus8.2606241549
cd "$REPO_ROOT"
export PYTHONPATH="$REPO_ROOT/src"
source /cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/Psi0-yth-v15/.venv-psi/bin/activate
[ -f .env ] && source .env
export PYTHONPATH="$REPO_ROOT/src"
export CUDA_VISIBLE_DEVICES=${CUDA_VISIBLE_DEVICES:-0}
export PSI_INFER_FLOAT32=1
pkill -f monitor.py || true
canonical_mode() {
case "$1" in
all|multi|full)
echo all
;;
text|pure_text)
echo text
;;
audio|audio_text|audio_only|audio_pure)
echo audio
;;
audio_command|audio_command_text|audio_command_only|command_audio)
echo audio_command
;;
video_clip|video_clip_text|video_clip_only)
echo video_clip
;;
video|video_text|video_only)
echo video
;;
*)
echo "$1"
;;
esac
}
RAW_MODE=${PSI_AUTO_INFER_MODE:-all}
MODE=$(canonical_mode "$RAW_MODE")
MODE_SEQUENCE=(text audio audio_command video_clip)
# MODE_SEQUENCE=(audio_command)
RUNS_ROOT=${RUNS_ROOT:-$REPO_ROOT/runs/posttrain_v1_trimodal}
INFER_PY=${INFER_PY:-/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/Psi0-yth-v15/.venv-psi/bin/python}
# RUN_DIR=${RUN_DIR:-auto}
RUN_DIR=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/Psi0-yth-v21/runs/posttrain_v1_trimodal/posttrain.sg.flow1000.cwml.lr1.0e-04.b512.gpus8.2607051648
if [ $# -ge 1 ]; then
RUN_DIR=$1
elif [ "$RUN_DIR" = "auto" ] || [ -z "$RUN_DIR" ]; then
AUTO_RUN_DIR=$("$INFER_PY" - "$MODE" "$RUNS_ROOT" <<'PY'
import json
import re
import sys
from pathlib import Path
mode = sys.argv[1]
runs_root = Path(sys.argv[2])
def ratio(data, key):
try:
return float(data.get(key, 0.0))
except Exception:
return 0.0
def supported_modes(run_config):
data = run_config.get("data", {})
modes = []
if ratio(data, "text_ratio") > 0 or "root_dir" in data:
modes.append("text")
if ratio(data, "audio_ratio") > 0 or ratio(data, "audio_only_ratio") > 0:
modes.append("audio")
if ratio(data, "audio_command_ratio") > 0 or ratio(data, "audio_command_only_ratio") > 0:
modes.append("audio_command")
if ratio(data, "video_clip_text_ratio") > 0 or ratio(data, "video_clip_only_ratio") > 0:
modes.append("video_clip")
if ratio(data, "video_ratio") > 0 or ratio(data, "video_only_ratio") > 0:
modes.append("video")
return modes
def run_sort_key(path):
match = re.search(r"(\d{10})$", path.name)
if match:
return int(match.group(1))
return int(path.stat().st_mtime)
candidates = []
for path in runs_root.glob("posttrain*.gpus8.*"):
config_path = path / "run_config.json"
if not config_path.exists():
continue
try:
run_config = json.loads(config_path.read_text())
except Exception:
continue
modes = supported_modes(run_config)
if mode == "all":
score = len([m for m in ("text", "audio", "audio_command", "video_clip", "video") if m in modes])
else:
score = 1 if mode in modes else 0
if score > 0:
candidates.append((score, run_sort_key(path), path))
if candidates:
candidates.sort()
print(str(candidates[-1][2]))
PY
)
if [ -n "$AUTO_RUN_DIR" ]; then
RUN_DIR=$AUTO_RUN_DIR
else
RUN_DIR=$DEFAULT_RUN_DIR
fi
fi
if [ -z "$RUN_DIR" ] || [ ! -f "$RUN_DIR/run_config.json" ]; then
echo "ERROR: cannot resolve a valid RUN_DIR for MODE=$MODE"
echo " Pass it explicitly: bash infer_render_auto.sh /path/to/run_dir"
exit 1
fi
SUPPORTED_MODES_CSV=$("$INFER_PY" - "$RUN_DIR" <<'PY'
import json
import sys
from pathlib import Path
run_dir = Path(sys.argv[1])
config = json.loads((run_dir / "run_config.json").read_text())
data = config.get("data", {})
def ratio(key):
try:
return float(data.get(key, 0.0))
except Exception:
return 0.0
supported = []
if ratio("text_ratio") > 0 or "root_dir" in data:
supported.append("text")
if ratio("audio_ratio") > 0 or ratio("audio_only_ratio") > 0:
supported.append("audio")
if ratio("audio_command_ratio") > 0 or ratio("audio_command_only_ratio") > 0:
supported.append("audio_command")
if ratio("video_clip_text_ratio") > 0 or ratio("video_clip_only_ratio") > 0:
supported.append("video_clip")
if ratio("video_ratio") > 0 or ratio("video_only_ratio") > 0:
supported.append("video")
print(",".join(supported))
PY
)
if [ -z "$SUPPORTED_MODES_CSV" ]; then
echo "ERROR: RUN_DIR does not expose any supported inference modes: $RUN_DIR"
exit 1
fi
IFS=',' read -r -a SUPPORTED_MODES <<< "$SUPPORTED_MODES_CSV"
mode_supported() {
local wanted
local supported
wanted=$(canonical_mode "$1")
for supported in "${SUPPORTED_MODES[@]}"; do
if [ "$supported" = "$wanted" ]; then
return 0
fi
done
return 1
}
TARGET_MODES=()
if [ "$MODE" = "all" ]; then
if mode_supported text; then
TARGET_MODES+=(text)
fi
if mode_supported audio; then
TARGET_MODES+=(audio)
fi
if mode_supported audio_command; then
TARGET_MODES+=(audio_command)
fi
if mode_supported video_clip; then
TARGET_MODES+=(video_clip)
elif mode_supported video; then
TARGET_MODES+=(video)
fi
if [ ${#TARGET_MODES[@]} -eq 0 ]; then
echo "ERROR: MODE=all but RUN_DIR has no supported target modes: $RUN_DIR"
exit 1
fi
else
if ! mode_supported "$RAW_MODE"; then
echo "ERROR: RUN_DIR does not look compatible with MODE=$RAW_MODE: $RUN_DIR"
echo " supported modes: ${SUPPORTED_MODES[*]}"
exit 1
fi
TARGET_MODES=("$RAW_MODE")
fi
CHECK_INTERVAL=10
NUM_SAMPLES=1
NUM_INFERENCE_STEPS=10
NUM_OUTPUT_FRAMES=500
AUDIO_NUM_OUTPUT_FRAMES=300
VIDEO_NUM_OUTPUT_FRAMES=300
VIDEO_CLIP_MOTION_FPS=50
DEVICE=cuda:0
GMR_PY=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/envs/gmr/bin/python
SONIC_RENDER_ROOT=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/GR00T-WholeBodyControl/gear_sonic
SOURCE_FPS=50
TARGET_FPS=30
TEXT_PROMPTS=(
"A person jogs forward at a steady pace."
"A person walks forward calmly."
"A person performs a high jump and lands steadily."
"A person turns around in place."
)
TEXT_PROMPT_NAMES=(
jog_forward
walk_forward
high_jump
turn_in_place
)
TEXT_INIT_MOTION_DIRS=(
/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/Psi0/dataset/seed_all/g1/csv_sonic/230710/jog_ff_loop_360_R_001__A419_M
/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/Psi0/dataset/seed_all/g1/csv_sonic/230710/walk_ff_loop_315_R_001__A421_M
/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/Psi0/dataset/seed_all/g1/csv_sonic/230710/high_jump_R_001__A418_M
/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/Psi0/dataset/seed_all/g1/csv_sonic/230710/idle_turn_360_R_001__A418_M
)
VIDEO_MOTION_DIRS=(
/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/GR00T-WholeBodyControl/gear_sonic_deploy/reference/converted/psi0_dance/gvhmr/movies__F-SNBR4jhY_0007
)
VIDEO_PATHS=(
/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/dance_1723_video_psi0/dance_1723_video_10s/movies__F-SNBR4jhY_0007.mp4
)
VIDEO_PROMPTS=(
"Generate a motion sequence that matches the provided video."
)
AUDIO_MOTION_DIRS=(
/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/GR00T-WholeBodyControl/gear_sonic_deploy/reference/converted/psi0_dance/gvhmr/movies__F-SNBR4jhY_0007
)
AUDIO_PATHS=(
/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/dance_1723_video_psi0/dance_1723_video_10s/movies__F-SNBR4jhY_0007.mp4
)
AUDIO_PROMPTS=(
"Generate a dance sequence that matches the provided music."
)
AUDIO_COMMAND_MOTION_DIRS=(
/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/Psi0/dataset/seed_all/g1/csv_sonic/230710/walk_ff_loop_315_R_001__A421_M
)
AUDIO_COMMAND_PATHS=(
/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/qwen3-tts/seed_audio/003771/female_01_zh.wav
)
AUDIO_COMMAND_PROMPTS=(
"Generate a motion that follows the spoken instruction in the audio."
)
RUN_PARENT_DIR=$(dirname "$RUN_DIR")
RUN_NAME=$(basename "$RUN_DIR")
RUN_TAG=$(echo "$RUN_NAME" | awk -F. '{print $NF}')
if [ ${#TARGET_MODES[@]} -gt 1 ]; then
OUTPUT_ROOT=${PSI_AUTO_OUTPUT_DIR:-"$RUN_PARENT_DIR/$RUN_TAG.auto_infer_all"}
else
OUTPUT_ROOT=${PSI_AUTO_OUTPUT_DIR:-"$RUN_PARENT_DIR/$RUN_TAG.auto_infer.${TARGET_MODES[0]}"}
fi
output_dir_for_mode() {
local mode=$1
if [ ${#TARGET_MODES[@]} -gt 1 ]; then
echo "$OUTPUT_ROOT/$(canonical_mode "$mode")"
else
echo "$OUTPUT_ROOT"
fi
}
expected_samples_for_mode() {
case "$1" in
text|pure_text)
echo "${#TEXT_PROMPTS[@]}"
;;
audio|audio_text|audio_only|audio_pure)
echo "${#AUDIO_MOTION_DIRS[@]}"
;;
audio_command|audio_command_text|audio_command_only|command_audio)
echo "${#AUDIO_COMMAND_MOTION_DIRS[@]}"
;;
video_clip|video_clip_text|video_clip_only|video|video_text|video_only)
echo "${#VIDEO_MOTION_DIRS[@]}"
;;
*)
echo 0
;;
esac
}
expected_motion_frames_for_mode() {
case "$1" in
text|pure_text)
echo "$NUM_OUTPUT_FRAMES"
;;
audio|audio_text|audio_only|audio_pure|audio_command|audio_command_text|audio_command_only|command_audio)
echo "$AUDIO_NUM_OUTPUT_FRAMES"
;;
video_clip|video_clip_text|video_clip_only|video|video_text|video_only)
echo "$VIDEO_NUM_OUTPUT_FRAMES"
;;
*)
echo "$NUM_OUTPUT_FRAMES"
;;
esac
}
min_render_frames_for_mode() {
local mode=$1
local motion_frames
motion_frames=$(expected_motion_frames_for_mode "$mode")
python -c 'import sys
motion_frames = int(sys.argv[1])
source_fps = float(sys.argv[2])
target_fps = float(sys.argv[3])
print(int(round((max(motion_frames, 1) - 1) / source_fps * target_fps)) + 1)' "$motion_frames" "$SOURCE_FPS" "$TARGET_FPS"
}
count_valid_render_mp4() {
local step_dir=$1
local min_render_frames=$2
local valid_render_mp4_count=0
local mp4_path
local mp4_frames
while IFS= read -r mp4_path; do
mp4_frames=$(ffprobe -v error -select_streams v:0 -show_entries stream=nb_frames -of default=noprint_wrappers=1:nokey=1 "$mp4_path" 2>/dev/null | head -n1)
if [ -n "$mp4_frames" ] && [ "$mp4_frames" -ge "$min_render_frames" ] 2>/dev/null; then
valid_render_mp4_count=$((valid_render_mp4_count + 1))
fi
done < <(find "$step_dir" -name 'reference_motion.mp4' -print 2>/dev/null)
echo "$valid_render_mp4_count"
}
run_infer_for_mode() {
local mode=$1
local ckpt_name=$2
local ckpt_step=$3
local step_dir=$4
local mode_key
local sample_i
local PROMPT
local PROMPT_NAME
local INIT_MOTION_DIR
local MOTION_DIR
local AUDIO_PATH
local VIDEO_PATH
local AUDIO_CONFIG_KEY
local AUDIO_MODE_NAME
local infer_failed=0
mode_key=$(canonical_mode "$mode")
case "$mode" in
text|pure_text)
for sample_i in "${!TEXT_PROMPTS[@]}"; do
PROMPT=${TEXT_PROMPTS[$sample_i]}
PROMPT_NAME=${TEXT_PROMPT_NAMES[$sample_i]}
INIT_MOTION_DIR=${TEXT_INIT_MOTION_DIRS[$sample_i]}
echo " [推理] 开始 $mode infer sample=$sample_i"
echo " prompt: $PROMPT"
echo " init : $INIT_MOTION_DIR"
"$INFER_PY" scripts/infer/seed_text_to_motion.py \
--run-dir "$RUN_DIR" \
--ckpt-step "$ckpt_step" \
--output-dir "$step_dir" \
--prompt "$PROMPT" \
--prompt-motion-name "$PROMPT_NAME" \
--init-motion-dir "$INIT_MOTION_DIR" \
--num-inference-steps "$NUM_INFERENCE_STEPS" \
--num-output-frames "$NUM_OUTPUT_FRAMES" \
--force-text-only \
--device "$DEVICE"
if [ $? -ne 0 ]; then
echo " [错误] $ckpt_name $mode infer失败,sample=$sample_i"
infer_failed=1
break
fi
done
;;
audio|audio_text|audio_only|audio_pure|audio_command|audio_command_text|audio_command_only|command_audio)
if [ "$mode_key" = "audio" ]; then
AUDIO_CONFIG_KEY=audio
AUDIO_MODE_NAME=audio
MOTION_DIRS=("${AUDIO_MOTION_DIRS[@]}")
AUDIO_FILES=("${AUDIO_PATHS[@]}")
PROMPTS=("${AUDIO_PROMPTS[@]}")
else
AUDIO_CONFIG_KEY=audio_command
AUDIO_MODE_NAME=audio_command
MOTION_DIRS=("${AUDIO_COMMAND_MOTION_DIRS[@]}")
AUDIO_FILES=("${AUDIO_COMMAND_PATHS[@]}")
PROMPTS=("${AUDIO_COMMAND_PROMPTS[@]}")
fi
for sample_i in "${!MOTION_DIRS[@]}"; do
MOTION_DIR=${MOTION_DIRS[$sample_i]}
AUDIO_PATH=${AUDIO_FILES[$sample_i]}
PROMPT=${PROMPTS[$sample_i]}
echo " [推理] 开始 $mode infer sample=$sample_i"
echo " audio_config: $AUDIO_CONFIG_KEY"
echo " motion : $MOTION_DIR"
echo " audio : $AUDIO_PATH"
"$INFER_PY" scripts/infer/seed_audio_to_motion.py \
--run-dir "$RUN_DIR" \
--ckpt-step "$ckpt_step" \
--output-dir "$step_dir" \
--motion-dir "$MOTION_DIR" \
--video-path "$AUDIO_PATH" \
--audio-config-key "$AUDIO_CONFIG_KEY" \
--prompt "$PROMPT" \
--num-inference-steps "$NUM_INFERENCE_STEPS" \
--num-output-frames "$AUDIO_NUM_OUTPUT_FRAMES" \
--device "$DEVICE"
if [ $? -ne 0 ]; then
echo " [错误] $ckpt_name $AUDIO_MODE_NAME infer失败,sample=$sample_i"
infer_failed=1
break
fi
done
;;
video_clip|video_clip_text|video_clip_only|video|video_text|video_only)
for sample_i in "${!VIDEO_MOTION_DIRS[@]}"; do
MOTION_DIR=${VIDEO_MOTION_DIRS[$sample_i]}
VIDEO_PATH=${VIDEO_PATHS[$sample_i]}
PROMPT=${VIDEO_PROMPTS[$sample_i]}
VIDEO_ARGS=(--video-config-key video)
if [ "$mode_key" = "video_clip" ]; then
VIDEO_ARGS=(--video-config-key video_clip --video-clip --video-clip-motion-fps "$VIDEO_CLIP_MOTION_FPS")
fi
if [ "$mode" = "video_only" ] || [ "$mode" = "video_clip_only" ]; then
PROMPT=""
fi
echo " [推理] 开始 $mode infer sample=$sample_i"
echo " motion: $MOTION_DIR"
echo " video : $VIDEO_PATH"
"$INFER_PY" scripts/infer/seed_video_to_motion.py \
--run-dir "$RUN_DIR" \
--ckpt-step "$ckpt_step" \
--output-dir "$step_dir" \
--motion-dir "$MOTION_DIR" \
--video-path "$VIDEO_PATH" \
--prompt "$PROMPT" \
--num-inference-steps "$NUM_INFERENCE_STEPS" \
--num-output-frames "$VIDEO_NUM_OUTPUT_FRAMES" \
"${VIDEO_ARGS[@]}" \
--device "$DEVICE"
if [ $? -ne 0 ]; then
echo " [错误] $ckpt_name $mode infer失败,sample=$sample_i"
infer_failed=1
break
fi
done
;;
*)
echo " [错误] unknown mode: $mode"
infer_failed=1
;;
esac
if [ "$infer_failed" -ne 0 ]; then
return 1
fi
return 0
}
run_render_for_mode() {
local mode=$1
local ckpt_name=$2
local step_dir=$3
local expected_samples=$4
local min_render_frames=$5
local valid_render_mp4_count
echo " [渲染] 开始 $mode render"
(
cd "$SONIC_RENDER_ROOT"
if [ -n "$XVFB_RUN" ]; then
env -u PYTHONPATH \
PYOPENGL_PLATFORM=egl \
MUJOCO_GL=egl \
QT_X11_NO_MITSHM=1 \
SONIC_RENDER_HEADLESS_CAMERA=1 \
SONIC_RENDER_LOCAL_TMP_OUTPUT=1 \
SONIC_RENDER_TMPDIR=/tmp \
"$XVFB_RUN" -a -s "-screen 0 1280x1024x24 -extension MIT-SHM" \
"$GMR_PY" scripts/batch_render_reference_motion_mujoco.py \
--motion-dir "$step_dir" \
--source-fps "$SOURCE_FPS" \
--target-fps "$TARGET_FPS" \
--dof-order isaaclab
else
echo " [渲染] xvfb-run not found; use headless EGL camera"
env -u PYTHONPATH \
PYOPENGL_PLATFORM=egl \
MUJOCO_GL=egl \
QT_X11_NO_MITSHM=1 \
SONIC_RENDER_HEADLESS_CAMERA=1 \
SONIC_RENDER_LOCAL_TMP_OUTPUT=1 \
SONIC_RENDER_TMPDIR=/tmp \
"$GMR_PY" scripts/batch_render_reference_motion_mujoco.py \
--motion-dir "$step_dir" \
--source-fps "$SOURCE_FPS" \
--target-fps "$TARGET_FPS" \
--dof-order isaaclab
fi
)
if [ $? -ne 0 ]; then
valid_render_mp4_count=$(count_valid_render_mp4 "$step_dir" "$min_render_frames")
if [ "$valid_render_mp4_count" -ge "$expected_samples" ]; then
echo " [警告] $mode render返回非0,但已生成 ${valid_render_mp4_count}/${expected_samples} 个有效reference_motion.mp4,视为完成"
else
echo " [错误] $ckpt_name $mode render失败,当前有效reference_motion.mp4数量: $valid_render_mp4_count/${expected_samples}"
return 1
fi
else
echo " [完成] $ckpt_name $mode render成功"
fi
return 0
}
process_mode_for_ckpt() {
local mode=$1
local ckpt_name=$2
local ckpt_step=$3
local output_dir
local step_dir
local expected_samples
local infer_csv_count
local min_render_frames
local valid_render_mp4_count
output_dir=$(output_dir_for_mode "$mode")
step_dir="$output_dir/step_$ckpt_step"
expected_samples=$(expected_samples_for_mode "$mode")
min_render_frames=$(min_render_frames_for_mode "$mode")
mkdir -p "$step_dir"
infer_csv_count=$(find "$step_dir" -name 'body_pos.csv' -print 2>/dev/null | wc -l | tr -d ' ')
valid_render_mp4_count=$(count_valid_render_mp4 "$step_dir" "$min_render_frames")
echo " [模式] $mode"
echo " 输出目录: $step_dir"
echo " infer: ${infer_csv_count}/${expected_samples}"
echo " render: ${valid_render_mp4_count}/${expected_samples}"
if [ "$infer_csv_count" -lt "$expected_samples" ]; then
if ! run_infer_for_mode "$mode" "$ckpt_name" "$ckpt_step" "$step_dir"; then
return 1
fi
infer_csv_count=$(find "$step_dir" -name 'body_pos.csv' -print 2>/dev/null | wc -l | tr -d ' ')
echo " [完成] $ckpt_name $mode infer成功,当前body_pos.csv数量: $infer_csv_count"
else
echo " [跳过] $mode infer已完成: ${infer_csv_count}/${expected_samples} 个body_pos.csv"
fi
valid_render_mp4_count=$(count_valid_render_mp4 "$step_dir" "$min_render_frames")
if [ "$valid_render_mp4_count" -lt "$expected_samples" ]; then
if ! run_render_for_mode "$mode" "$ckpt_name" "$step_dir" "$expected_samples" "$min_render_frames"; then
return 1
fi
else
echo " [跳过] $mode render已完成: ${valid_render_mp4_count}/${expected_samples} 个有效reference_motion.mp4"
fi
return 0
}
if [ -x /usr/bin/xvfb-run ]; then
XVFB_RUN=/usr/bin/xvfb-run
elif [ -x /bin/xvfb-run ]; then
XVFB_RUN=/bin/xvfb-run
elif command -v xvfb-run >/dev/null 2>&1; then
XVFB_RUN=$(command -v xvfb-run)
else
XVFB_RUN=
fi
export PYOPENGL_PLATFORM=egl
export MUJOCO_GL=egl
export QT_X11_NO_MITSHM=1
echo "=========================================="
echo "Auto Infer + Render Monitor"
echo "=========================================="
echo "RUN_DIR: $RUN_DIR"
echo "OUTPUT_ROOT: $OUTPUT_ROOT"
echo "RAW_MODE: $RAW_MODE"
echo "MODE: $MODE"
echo "SUPPORTED_MODES: ${SUPPORTED_MODES[*]}"
echo "TARGET_MODES: ${TARGET_MODES[*]}"
echo "CHECK_INTERVAL: ${CHECK_INTERVAL}s"
echo "NUM_OUTPUT_FRAMES: $NUM_OUTPUT_FRAMES"
echo "AUDIO_NUM_OUTPUT_FRAMES: $AUDIO_NUM_OUTPUT_FRAMES"
echo "VIDEO_NUM_OUTPUT_FRAMES: $VIDEO_NUM_OUTPUT_FRAMES"
echo "NUM_INFERENCE_STEPS: $NUM_INFERENCE_STEPS"
echo "XVFB_RUN: ${XVFB_RUN:-<not found, use headless EGL camera>}"
echo "NOTE: default MODE=all. Set PSI_AUTO_INFER_MODE to run a single modality."
echo "NOTE: single-mode values: text / audio / audio_command / video_clip"
echo "=========================================="
mkdir -p "$OUTPUT_ROOT"
for mode in "${TARGET_MODES[@]}"; do
mode_output_dir=$(output_dir_for_mode "$mode")
mkdir -p "$mode_output_dir"
echo "OUTPUT_DIR[$mode]: $mode_output_dir"
done
while true; do
echo ""
echo "[$(date '+%Y-%m-%d %H:%M:%S')] 扫描新的checkpoint..."
pending_ckpts=()
checkpoint_dirs=()
for ckpt_dir in "$RUN_DIR"/checkpoints/ckpt_*; do
[ -d "$ckpt_dir" ] || continue
checkpoint_dirs+=("$ckpt_dir")
done
if [ ${#checkpoint_dirs[@]} -eq 0 ] && [ -d "$RUN_DIR/checkpoints/best_ckpt" ]; then
checkpoint_dirs+=("$RUN_DIR/checkpoints/best_ckpt")
fi
for ckpt_dir in "${checkpoint_dirs[@]}"; do
ckpt_name=$(basename "$ckpt_dir")
if [ "$ckpt_name" = "best_ckpt" ]; then
ckpt_step=best
sort_step=999999999999
elif [[ "$ckpt_name" =~ ckpt_([0-9]+) ]]; then
ckpt_step="${BASH_REMATCH[1]}"
sort_step="$ckpt_step"
else
echo " [警告] 无法从 $ckpt_name 提取step数,跳过"
continue
fi
ckpt_pending_modes=()
for scan_mode in "${TARGET_MODES[@]}"; do
scan_step_dir="$(output_dir_for_mode "$scan_mode")/step_$ckpt_step"
scan_expected_samples=$(expected_samples_for_mode "$scan_mode")
scan_min_render_frames=$(min_render_frames_for_mode "$scan_mode")
scan_valid_render_mp4_count=$(count_valid_render_mp4 "$scan_step_dir" "$scan_min_render_frames")
if [ "$scan_valid_render_mp4_count" -lt "$scan_expected_samples" ]; then
ckpt_pending_modes+=("$scan_mode")
fi
done
if [ ${#ckpt_pending_modes[@]} -eq 0 ]; then
echo " [跳过] $ckpt_name (所有目标模态都已完成)"
continue
fi
pending_modes_csv=$(IFS=,; echo "${ckpt_pending_modes[*]}")
pending_ckpts+=("${sort_step}|${ckpt_step}|${ckpt_dir}|${pending_modes_csv}")
done
if [ ${#pending_ckpts[@]} -eq 0 ]; then
echo " [无] 没有待处理的checkpoint"
echo "[$(date '+%Y-%m-%d %H:%M:%S')] 等待 ${CHECK_INTERVAL} 秒后再次检查..."
sleep "$CHECK_INTERVAL"
continue
fi
best_item=$(printf '%s\n' "${pending_ckpts[@]}" | sort -t'|' -k1,1nr | head -n1)
IFS='|' read -r sort_step ckpt_step ckpt_dir pending_modes_csv <<< "$best_item"
ckpt_name=$(basename "$ckpt_dir")
IFS=',' read -r -a CKPT_PENDING_MODES <<< "$pending_modes_csv"
echo " [发现] 待处理checkpoint共 ${#pending_ckpts[@]} 个,选择step最大的:"
echo " $ckpt_name"
echo " [步骤] step: $ckpt_step"
echo " [模式] 待处理: ${CKPT_PENDING_MODES[*]}"
mode_failed=0
for mode in "${CKPT_PENDING_MODES[@]}"; do
if ! process_mode_for_ckpt "$mode" "$ckpt_name" "$ckpt_step"; then
mode_failed=1
break
fi
done
if [ "$mode_failed" -ne 0 ]; then
echo "[$(date '+%Y-%m-%d %H:%M:%S')] 等待 ${CHECK_INTERVAL} 秒后再次检查..."
sleep "$CHECK_INTERVAL"
continue
fi
echo "[$(date '+%Y-%m-%d %H:%M:%S')] 等待 ${CHECK_INTERVAL} 秒后再次检查..."
sleep "$CHECK_INTERVAL"
done