#!/usr/bin/env bash set -Eeuo pipefail # Internal audio-case/render worker. Public entry: infer_render_auto.sh. PROJECT_ROOT=$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd) cd "$PROJECT_ROOT" export PYTHONPATH="$PROJECT_ROOT/src${PYTHONPATH:+:$PYTHONPATH}" source "$PROJECT_ROOT/scripts/activate_psi_env.sh" HFD_ENV_FILE=${HFD_ENV_FILE:-$PROJECT_ROOT/.env} if [[ -f "$HFD_ENV_FILE" ]]; then set -a source "$HFD_ENV_FILE" set +a fi PYTHON_BIN=${HFD_INFER_PYTHON:-python} RUNS_ROOT=${HFD_INFER_RUNS_ROOT:-$PROJECT_ROOT/runs} RUN_DIR=${HFD_INFER_RUN_DIR:-auto} if [[ $# -gt 0 ]]; then RUN_DIR=$1 fi if [[ "$RUN_DIR" == "auto" ]]; then RUN_DIR=$($PYTHON_BIN - "$RUNS_ROOT" <<'PY' import json import sys from pathlib import Path candidates = [] for config_path in Path(sys.argv[1]).rglob("run_config.json"): try: config = json.loads(config_path.read_text(encoding="utf-8")) except Exception: continue model = config.get("model", {}) data = config.get("data", {}) train = config.get("train", {}) if ( not model.get("streaming_qwen_enabled", False) or not data.get("online_audio", False) or train.get("full_duplex_semantics_version") not in {9, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24} ): continue candidates.append((config_path.stat().st_mtime_ns, config_path.parent)) if candidates: print(max(candidates, key=lambda item: item[0])[1]) PY ) fi if [[ -z "$RUN_DIR" || ! -f "$RUN_DIR/run_config.json" ]]; then echo "[error] Cannot find a full-duplex RUN_DIR. Pass it as the first argument." >&2 exit 2 fi RUN_DIR=$(cd "$RUN_DIR" && pwd) if [[ -n "${HFD_INFER_GPUS:-}" ]]; then export CUDA_VISIBLE_DEVICES=$HFD_INFER_GPUS elif [[ -z "${CUDA_VISIBLE_DEVICES:-}" ]]; then export CUDA_VISIBLE_DEVICES=0 fi DEVICE=${HFD_INFER_DEVICE:-cuda:0} CHECK_INTERVAL=${HFD_INFER_CHECK_INTERVAL:-30} INFER_ONCE=${HFD_INFER_ONCE:-0} MOTION_FRAMES=${HFD_INFER_MOTION_FRAMES:-450} INFERENCE_STEPS=${HFD_INFER_STEPS:-10} PROMPT=${HFD_INFER_PROMPT:-auto} RESPECT_CONTROL=${HFD_INFER_RESPECT_CONTROL:-1} MAX_LISTEN_TAIL_STEPS=${HFD_INFER_MAX_LISTEN_TAIL_STEPS:-500} INITIAL_IDLE_STEPS=${HFD_INFER_INITIAL_IDLE_STEPS:-5} WHITE_NOISE_START_INDEX=${HFD_INFER_WHITE_NOISE_START_INDEX:-3} OUTPUT_SUBDIR=${HFD_INFER_OUTPUT_SUBDIR:-full_duplex_auto_infer} OUTPUT_ROOT=${HFD_INFER_OUTPUT_ROOT:-$RUN_DIR/$OUTPUT_SUBDIR} GMR_PY=${HFD_RENDER_PYTHON:-/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/envs/gmr/bin/python} GMR_XML=${HFD_RENDER_XML:-/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/GMR/assets/unitree_g1/g1_mocap_29dof.xml} SONIC_RENDER_ROOT=${HFD_RENDER_ROOT:-/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/GR00T-WholeBodyControl/gear_sonic} RENDER_FPS=${HFD_RENDER_FPS:-50} TERMINAL_BACKGROUND_STEPS=${HFD_INFER_TERMINAL_BACKGROUND_STEPS:-0} INTER_TURN_BACKGROUND_STEPS_BY_TURN=${HFD_INFER_INTER_TURN_BACKGROUND_STEPS_BY_TURN:-} MIN_CHECKPOINT_STEP=${HFD_INFER_MIN_CHECKPOINT_STEP:-0} EXACT_CHECKPOINT_STEP=${HFD_INFER_EXACT_CHECKPOINT_STEP:-} SAMPLE_TAG=${HFD_INFER_SAMPLE_TAG:-} SAMPLE_PARENT=${HFD_INFER_SAMPLE_PARENT:-} SCENARIO_FILE=${HFD_INFER_SCENARIO_FILE:-} if [[ -n "$EXACT_CHECKPOINT_STEP" && ! "$EXACT_CHECKPOINT_STEP" =~ ^[0-9]+$ ]]; then echo "[error] HFD_INFER_EXACT_CHECKPOINT_STEP must be a non-negative integer." >&2 exit 2 fi if [[ -n "$SAMPLE_PARENT" && ! "$SAMPLE_PARENT" =~ ^[[:alnum:]_.-]+$ ]]; then echo "[error] HFD_INFER_SAMPLE_PARENT must be one safe path component." >&2 exit 2 fi # Default sample is one of the 50 fixed overfit sources (manifest row 48, # "character walks forward while grabbing something"), using the exact fixed # voice the trainer binds to that row (row_id 48 % 8 voices = female_01_en) # and a verified neutral-idle motion as the initial pose. Starting directly # from the task GT first frame is outside the training construction, which # always begins from an audited upright stationary anchor and learns a 20-frame # transition into the task. The previous default # (003771, walk back-left) is not in the overfit set: a memorizing model has # never seen a back-left walk and can only replay the forward walk it knows. DEFAULT_AUDIO=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/qwen3-tts/seed_audio/000006/female_01_en.wav DEFAULT_INIT_MOTION=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/Psi0/dataset/seed_all/g1/csv_sonic/221125/neutral_idle_loop_002__A073 AUDIO_PATHS_CSV=${HFD_INFER_AUDIO_PATHS:-$DEFAULT_AUDIO} ADDITIONAL_AUDIO_PATHS_CSV=${HFD_INFER_ADDITIONAL_AUDIO_PATHS:-} INTER_TURN_BACKGROUND_STEPS=${HFD_INFER_INTER_TURN_BACKGROUND_STEPS:-0} INIT_MOTION_DIRS_CSV=${HFD_INFER_INIT_MOTION_DIRS:-$DEFAULT_INIT_MOTION} IFS=',' read -r -a AUDIO_PATHS <<< "$AUDIO_PATHS_CSV" ADDITIONAL_AUDIO_PATHS=() if [[ -n "$ADDITIONAL_AUDIO_PATHS_CSV" ]]; then IFS=',' read -r -a ADDITIONAL_AUDIO_PATHS <<< "$ADDITIONAL_AUDIO_PATHS_CSV" fi IFS=',' read -r -a INIT_MOTION_DIRS <<< "$INIT_MOTION_DIRS_CSV" if [[ ${#INIT_MOTION_DIRS[@]} -ne 1 && ${#INIT_MOTION_DIRS[@]} -ne ${#AUDIO_PATHS[@]} ]]; then echo "[error] HFD_INFER_INIT_MOTION_DIRS must contain one path or match HFD_INFER_AUDIO_PATHS." >&2 exit 2 fi checkpoint_ready() { local checkpoint_dir=$1 "$PYTHON_BIN" - "$checkpoint_dir" <<'PY' import json import sys from pathlib import Path checkpoint = Path(sys.argv[1]) single = checkpoint / "model.safetensors" manifest = checkpoint / "hfd_fused_delta_manifest.json" boundary = checkpoint / "full_duplex_source_boundary.json" if ( single.is_file() and single.stat().st_size > 0 and manifest.is_file() and manifest.stat().st_size > 0 and boundary.is_file() and boundary.stat().st_size > 0 ): payload = json.loads(boundary.read_text(encoding="utf-8")) if bool(payload.get("source_group_complete", False)): raise SystemExit(0) raise SystemExit(1) PY } checkpoint_step() { local checkpoint_name=$1 if [[ "$checkpoint_name" == "best_ckpt" ]]; then echo best else echo "${checkpoint_name#ckpt_}" fi } checkpoint_output_name() { local checkpoint_dir=$1 local captured_global_step=${2:-} local checkpoint_name checkpoint_name=$(basename "$checkpoint_dir") if [[ "$checkpoint_name" != "best_ckpt" ]]; then echo "$checkpoint_name" return fi if [[ -z "$captured_global_step" ]]; then captured_global_step=$(checkpoint_global_step "$checkpoint_dir") fi echo "best_ckpt_step_${captured_global_step}" } checkpoint_global_step() { local checkpoint_dir=$1 "$PYTHON_BIN" - "$checkpoint_dir/full_duplex_source_boundary.json" <<'PY' import json import sys from pathlib import Path marker_path = Path(sys.argv[1]) if not marker_path.is_file(): raise SystemExit(f"checkpoint has no source-boundary marker: {marker_path}") payload = json.loads(marker_path.read_text(encoding="utf-8")) step = int(payload["global_step"]) if step <= 0: raise SystemExit(f"checkpoint has invalid global_step={step}") print(step) PY } safe_sample_name() { local audio_path=$1 local index=$2 local stem stem=$(basename "$audio_path") stem=${stem%.*} stem=$(echo "$stem" | tr -cs '[:alnum:]_.-' '_') if [[ -n "$SAMPLE_TAG" ]]; then stem="${SAMPLE_TAG}_${stem}" fi if [[ ${#ADDITIONAL_AUDIO_PATHS[@]} -gt 0 ]]; then printf '%02d_%s_session%dturn' \ "$index" "$stem" "$(( ${#ADDITIONAL_AUDIO_PATHS[@]} + 1 ))" else printf '%02d_%s' "$index" "$stem" fi } sample_output_dir() { local checkpoint_name=$1 local sample_name=$2 if [[ -n "$SAMPLE_PARENT" ]]; then printf '%s/%s/%s/%s\n' \ "$OUTPUT_ROOT" "$checkpoint_name" "$SAMPLE_PARENT" "$sample_name" else printf '%s/%s/%s\n' "$OUTPUT_ROOT" "$checkpoint_name" "$sample_name" fi } render_sample() { local sample_dir=$1 ( cd "$SONIC_RENDER_ROOT" env -u PYTHONPATH \ PYOPENGL_PLATFORM=egl \ MUJOCO_GL=egl \ QT_X11_NO_MITSHM=1 \ SONIC_RENDER_HEADLESS_CAMERA=1 \ SONIC_RENDER_LOCAL_TMP_OUTPUT=1 \ SONIC_RENDER_TMPDIR=/tmp \ "$GMR_PY" scripts/batch_render_reference_motion_mujoco.py \ --motion-dir "$sample_dir" \ --xml-path "$GMR_XML" \ --source-fps 50 \ --target-fps "$RENDER_FPS" \ --dof-order isaaclab ) } process_sample() { local checkpoint_name=$1 local step=$2 local sample_index=$3 local expected_global_step=$4 local audio_path=${AUDIO_PATHS[$sample_index]} local init_index=$sample_index if [[ ${#INIT_MOTION_DIRS[@]} -eq 1 ]]; then init_index=0 fi local init_motion_dir=${INIT_MOTION_DIRS[$init_index]} local sample_name sample_name=$(safe_sample_name "$audio_path" "$sample_index") local sample_dir sample_dir=$(sample_output_dir "$checkpoint_name" "$sample_name") mkdir -p "$sample_dir" if [[ ! -s "$sample_dir/infer_meta.json" || ! -s "$sample_dir/body_pos.csv" ]]; then echo "[infer] checkpoint=$checkpoint_name sample=$sample_name audio=$audio_path" control_args=() additional_audio_args=() inter_turn_args=() for additional_audio_path in "${ADDITIONAL_AUDIO_PATHS[@]}"; do additional_audio_args+=(--additional-audio-path "$additional_audio_path") done if [[ -n "$INTER_TURN_BACKGROUND_STEPS_BY_TURN" ]]; then inter_turn_args+=( --inter-turn-background-steps-by-turn "$INTER_TURN_BACKGROUND_STEPS_BY_TURN" ) fi if [[ "$RESPECT_CONTROL" != "1" && "$RESPECT_CONTROL" != "true" ]]; then control_args+=(--ignore-control-token) fi rm -f "$sample_dir/infer_failure.json" if "$PYTHON_BIN" -u scripts/infer/full_duplex_audio_to_motion.py \ --run-dir "$RUN_DIR" \ --ckpt-step "$step" \ --output-dir "$sample_dir" \ --audio-path "$audio_path" \ "${additional_audio_args[@]}" \ --init-motion-dir "$init_motion_dir" \ --motion-frames "$MOTION_FRAMES" \ --inter-turn-background-steps "$INTER_TURN_BACKGROUND_STEPS" \ "${inter_turn_args[@]}" \ --terminal-background-steps "$TERMINAL_BACKGROUND_STEPS" \ --num-inference-steps "$INFERENCE_STEPS" \ --prompt "$PROMPT" \ --max-listen-tail-steps "$MAX_LISTEN_TAIL_STEPS" \ --initial-idle-steps "$INITIAL_IDLE_STEPS" \ --white-noise-start-index "$WHITE_NOISE_START_INDEX" \ --expected-checkpoint-global-step "$expected_global_step" \ --device "$DEVICE" \ "${control_args[@]}" 2>&1 | tee "$sample_dir/infer.log"; then : else local infer_status=${PIPESTATUS[0]} return "$infer_status" fi else echo "[skip] inference exists: $sample_dir" fi if [[ "${HFD_INFER_SKIP_RENDER:-0}" == "1" ]]; then touch "$sample_dir/.inferred" echo "[inferred] queued independent rendering: $sample_dir" return 0 fi render_and_mux_sample "$sample_dir" } render_and_mux_sample() { local sample_dir=$1 if [[ ! -f "$sample_dir/.complete" ]]; then # The renderer validates the frame count of an existing intermediate MP4. # A cancelled render/mux must never be accepted merely because it is nonempty. echo "[render] sample=$sample_dir fps=$RENDER_FPS" if ! render_sample "$sample_dir"; then echo "[error] render failed: $sample_dir" >&2 return 1 fi else echo "[skip] render exists: $sample_dir/reference_motion.mp4" fi if [[ ! -f "$sample_dir/.complete" ]]; then echo "[mux] use the complete background/speech/background streaming audio timeline" if ! "$PYTHON_BIN" scripts/infer/mux_full_duplex_audio.py \ --video "$sample_dir/reference_motion.mp4" \ --audio "$sample_dir/assembled_input_audio_16k.wav" \ --output "$sample_dir/reference_motion_with_audio.mp4" \ --infer-meta "$sample_dir/infer_meta.json" \ --audio-is-timeline \ --expected-fps "$RENDER_FPS"; then echo "[error] audio mux failed: $sample_dir" >&2 return 1 fi else echo "[skip] synchronized video exists: $sample_dir/reference_motion_with_audio.mp4" fi touch "$sample_dir/.complete" } if [[ -n "${HFD_RENDER_ONLY_DIR:-}" ]]; then sample_dir=$HFD_RENDER_ONLY_DIR if [[ "${HFD_RENDER_TASK:-}" == "swa" ]]; then # Separate CPU-only preparation/composition; never reload a checkpoint. "$PYTHON_BIN" scripts/infer/render_swa_result.py prepare \ --run-dir "$RUN_DIR" --output-dir "$sample_dir" render_sample "$sample_dir/render_motion" "$PYTHON_BIN" scripts/infer/render_swa_result.py compose \ --run-dir "$RUN_DIR" --output-dir "$sample_dir" touch "$sample_dir/.complete" exit 0 fi for required_file in infer_meta.json body_pos.csv body_quat.csv joint_pos.csv assembled_input_audio_16k.wav; do if [[ ! -s "$sample_dir/$required_file" ]]; then echo "[error] render-only input missing: $sample_dir/$required_file" >&2 exit 1 fi done render_and_mux_sample "$sample_dir" exit 0 fi mkdir -p "$OUTPUT_ROOT" if [[ -n "$SCENARIO_FILE" ]]; then if [[ ! -f "$SCENARIO_FILE" ]]; then echo "[error] HFD_INFER_SCENARIO_FILE does not exist: $SCENARIO_FILE" >&2 exit 2 fi scenario_snapshot="$OUTPUT_ROOT/scenario.snapshot.json" if [[ -n "$SAMPLE_PARENT" ]]; then scenario_snapshot="$OUTPUT_ROOT/scenario.${SAMPLE_PARENT}.snapshot.json" fi cp -- "$SCENARIO_FILE" "$scenario_snapshot" fi if [[ "$MIN_CHECKPOINT_STEP" == "auto" ]]; then MIN_CHECKPOINT_STEP=0 if [[ -z "$EXACT_CHECKPOINT_STEP" ]]; then while IFS= read -r candidate_checkpoint; do checkpoint_ready "$candidate_checkpoint" || continue candidate_name=$(basename "$candidate_checkpoint") candidate_step=${candidate_name#ckpt_} if (( candidate_step > MIN_CHECKPOINT_STEP )); then MIN_CHECKPOINT_STEP=$candidate_step fi done < <( find "$RUN_DIR/checkpoints" -mindepth 1 -maxdepth 1 -type d \ -name 'ckpt_[0-9]*' -print 2>/dev/null ) fi fi if [[ ! "$MIN_CHECKPOINT_STEP" =~ ^[0-9]+$ ]]; then echo "[error] HFD_INFER_MIN_CHECKPOINT_STEP must be auto or a non-negative integer." >&2 exit 2 fi echo "RUN_DIR: $RUN_DIR" echo "OUTPUT_ROOT: $OUTPUT_ROOT" echo "TIMING: audio 16kHz, 0.6s window, 0.4s stride; action 30/20/10 at 50fps" echo "RENDER: ${RENDER_FPS}fps; speech follows ${INITIAL_IDLE_STEPS} idle windows; motion begins at recorded frame boundary" echo "CONTROL: respect=$RESPECT_CONTROL initial_idle_steps=$INITIAL_IDLE_STEPS max_listen_tail_steps=$MAX_LISTEN_TAIL_STEPS white_noise_start=$WHITE_NOISE_START_INDEX" echo "SESSION: primary_samples=${#AUDIO_PATHS[@]} additional_turns=${#ADDITIONAL_AUDIO_PATHS[@]} inter_turn_background_steps=$INTER_TURN_BACKGROUND_STEPS by_turn=${INTER_TURN_BACKGROUND_STEPS_BY_TURN:-uniform} terminal_background_steps=$TERMINAL_BACKGROUND_STEPS" echo "CHECKPOINT POLICY: immutable ckpt_ only; mutable best_ckpt is never watched; min_step=$MIN_CHECKPOINT_STEP exact_step=${EXACT_CHECKPOINT_STEP:-none}" while true; do if [[ -n "$EXACT_CHECKPOINT_STEP" ]]; then CHECKPOINT_DIRS=("$RUN_DIR/checkpoints/ckpt_$EXACT_CHECKPOINT_STEP") else mapfile -t CHECKPOINT_DIRS < <( find "$RUN_DIR/checkpoints" -mindepth 1 -maxdepth 1 -type d \ -name 'ckpt_[0-9]*' -print 2>/dev/null | sort -V ) fi pending=() for checkpoint_dir in "${CHECKPOINT_DIRS[@]}"; do [[ -d "$checkpoint_dir" ]] || continue checkpoint_dir_name=$(basename "$checkpoint_dir") checkpoint_number=${checkpoint_dir_name#ckpt_} if (( checkpoint_number < MIN_CHECKPOINT_STEP )); then continue fi checkpoint_ready "$checkpoint_dir" || continue if ! captured_global_step=$(checkpoint_global_step "$checkpoint_dir"); then echo "[wait] checkpoint identity is not ready: $checkpoint_dir" >&2 continue fi checkpoint_name=$(checkpoint_output_name "$checkpoint_dir" "$captured_global_step") settled=1 for sample_index in "${!AUDIO_PATHS[@]}"; do sample_name=$(safe_sample_name "${AUDIO_PATHS[$sample_index]}" "$sample_index") sample_dir=$(sample_output_dir "$checkpoint_name" "$sample_name") if [[ ! -f "$sample_dir/.complete" && ! -f "$sample_dir/.failed" ]]; then settled=0 break fi done [[ $settled -eq 1 ]] || pending+=("$checkpoint_dir") done if [[ ${#pending[@]} -eq 0 ]]; then echo "[$(date '+%F %T')] no pending checkpoint" if [[ "$INFER_ONCE" == "1" || "$INFER_ONCE" == "true" ]]; then break fi sleep "$CHECK_INTERVAL" continue fi checkpoint_dir=${pending[-1]} checkpoint_dir_name=$(basename "$checkpoint_dir") if ! expected_global_step=$(checkpoint_global_step "$checkpoint_dir"); then echo "[wait] selected checkpoint identity changed before inference: $checkpoint_dir" >&2 continue fi checkpoint_name=$(checkpoint_output_name "$checkpoint_dir" "$expected_global_step") step=$(checkpoint_step "$checkpoint_dir_name") echo "[$(date '+%F %T')] process newest pending checkpoint: $checkpoint_name" for sample_index in "${!AUDIO_PATHS[@]}"; do sample_name=$(safe_sample_name "${AUDIO_PATHS[$sample_index]}" "$sample_index") sample_dir=$(sample_output_dir "$checkpoint_name" "$sample_name") if [[ -f "$sample_dir/.complete" ]]; then echo "[skip] completed sample: $sample_dir" continue fi if [[ -f "$sample_dir/.failed" ]]; then echo "[skip] failed sample already recorded: $sample_dir" continue fi if process_sample \ "$checkpoint_name" \ "$step" \ "$sample_index" \ "$expected_global_step"; then continue else status=$? printf \ 'checkpoint=%s\nsample=%s\nexit_status=%s\nfailed_at=%s\n' \ "$checkpoint_name" \ "$sample_name" \ "$status" \ "$(date --iso-8601=seconds)" \ > "$sample_dir/.failed" echo "[failed] checkpoint=$checkpoint_name sample=$sample_name status=$status; details: $sample_dir/infer.log" >&2 fi done if [[ "$INFER_ONCE" == "1" || "$INFER_ONCE" == "true" ]]; then break fi done echo "[done] automatic full-duplex inference/render loop exited cleanly"