code-HFD / scripts /infer /run_audio_case.sh
Tinghong-Ye22's picture
Add files using upload-large-folder tool
06f9561 verified
Raw History Blame Contribute Delete
18.4 kB
#!/usr/bin/env bash
set -Eeuo pipefail
# Internal audio-case/render worker. Public entry: infer_render_auto.sh.
PROJECT_ROOT=$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)
cd "$PROJECT_ROOT"
export PYTHONPATH="$PROJECT_ROOT/src${PYTHONPATH:+:$PYTHONPATH}"
source "$PROJECT_ROOT/scripts/activate_psi_env.sh"
HFD_ENV_FILE=${HFD_ENV_FILE:-$PROJECT_ROOT/.env}
if [[ -f "$HFD_ENV_FILE" ]]; then
set -a
source "$HFD_ENV_FILE"
set +a
fi
PYTHON_BIN=${HFD_INFER_PYTHON:-python}
RUNS_ROOT=${HFD_INFER_RUNS_ROOT:-$PROJECT_ROOT/runs}
RUN_DIR=${HFD_INFER_RUN_DIR:-auto}
if [[ $# -gt 0 ]]; then
RUN_DIR=$1
fi
if [[ "$RUN_DIR" == "auto" ]]; then
RUN_DIR=$($PYTHON_BIN - "$RUNS_ROOT" <<'PY'
import json
import sys
from pathlib import Path
candidates = []
for config_path in Path(sys.argv[1]).rglob("run_config.json"):
try:
config = json.loads(config_path.read_text(encoding="utf-8"))
except Exception:
continue
model = config.get("model", {})
data = config.get("data", {})
train = config.get("train", {})
if (
not model.get("streaming_qwen_enabled", False)
or not data.get("online_audio", False)
or train.get("full_duplex_semantics_version") not in {9, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24}
):
continue
candidates.append((config_path.stat().st_mtime_ns, config_path.parent))
if candidates:
print(max(candidates, key=lambda item: item[0])[1])
PY
)
fi
if [[ -z "$RUN_DIR" || ! -f "$RUN_DIR/run_config.json" ]]; then
echo "[error] Cannot find a full-duplex RUN_DIR. Pass it as the first argument." >&2
exit 2
fi
RUN_DIR=$(cd "$RUN_DIR" && pwd)
if [[ -n "${HFD_INFER_GPUS:-}" ]]; then
export CUDA_VISIBLE_DEVICES=$HFD_INFER_GPUS
elif [[ -z "${CUDA_VISIBLE_DEVICES:-}" ]]; then
export CUDA_VISIBLE_DEVICES=0
fi
DEVICE=${HFD_INFER_DEVICE:-cuda:0}
CHECK_INTERVAL=${HFD_INFER_CHECK_INTERVAL:-30}
INFER_ONCE=${HFD_INFER_ONCE:-0}
MOTION_FRAMES=${HFD_INFER_MOTION_FRAMES:-450}
INFERENCE_STEPS=${HFD_INFER_STEPS:-10}
PROMPT=${HFD_INFER_PROMPT:-auto}
RESPECT_CONTROL=${HFD_INFER_RESPECT_CONTROL:-1}
MAX_LISTEN_TAIL_STEPS=${HFD_INFER_MAX_LISTEN_TAIL_STEPS:-500}
INITIAL_IDLE_STEPS=${HFD_INFER_INITIAL_IDLE_STEPS:-5}
WHITE_NOISE_START_INDEX=${HFD_INFER_WHITE_NOISE_START_INDEX:-3}
OUTPUT_SUBDIR=${HFD_INFER_OUTPUT_SUBDIR:-full_duplex_auto_infer}
OUTPUT_ROOT=${HFD_INFER_OUTPUT_ROOT:-$RUN_DIR/$OUTPUT_SUBDIR}
GMR_PY=${HFD_RENDER_PYTHON:-/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/envs/gmr/bin/python}
GMR_XML=${HFD_RENDER_XML:-/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/GMR/assets/unitree_g1/g1_mocap_29dof.xml}
SONIC_RENDER_ROOT=${HFD_RENDER_ROOT:-/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/GR00T-WholeBodyControl/gear_sonic}
RENDER_FPS=${HFD_RENDER_FPS:-50}
TERMINAL_BACKGROUND_STEPS=${HFD_INFER_TERMINAL_BACKGROUND_STEPS:-0}
INTER_TURN_BACKGROUND_STEPS_BY_TURN=${HFD_INFER_INTER_TURN_BACKGROUND_STEPS_BY_TURN:-}
MIN_CHECKPOINT_STEP=${HFD_INFER_MIN_CHECKPOINT_STEP:-0}
EXACT_CHECKPOINT_STEP=${HFD_INFER_EXACT_CHECKPOINT_STEP:-}
SAMPLE_TAG=${HFD_INFER_SAMPLE_TAG:-}
SAMPLE_PARENT=${HFD_INFER_SAMPLE_PARENT:-}
SCENARIO_FILE=${HFD_INFER_SCENARIO_FILE:-}
if [[ -n "$EXACT_CHECKPOINT_STEP" && ! "$EXACT_CHECKPOINT_STEP" =~ ^[0-9]+$ ]]; then
echo "[error] HFD_INFER_EXACT_CHECKPOINT_STEP must be a non-negative integer." >&2
exit 2
fi
if [[ -n "$SAMPLE_PARENT" && ! "$SAMPLE_PARENT" =~ ^[[:alnum:]_.-]+$ ]]; then
echo "[error] HFD_INFER_SAMPLE_PARENT must be one safe path component." >&2
exit 2
fi
# Default sample is one of the 50 fixed overfit sources (manifest row 48,
# "character walks forward while grabbing something"), using the exact fixed
# voice the trainer binds to that row (row_id 48 % 8 voices = female_01_en)
# and a verified neutral-idle motion as the initial pose. Starting directly
# from the task GT first frame is outside the training construction, which
# always begins from an audited upright stationary anchor and learns a 20-frame
# transition into the task. The previous default
# (003771, walk back-left) is not in the overfit set: a memorizing model has
# never seen a back-left walk and can only replay the forward walk it knows.
DEFAULT_AUDIO=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/qwen3-tts/seed_audio/000006/female_01_en.wav
DEFAULT_INIT_MOTION=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/Psi0/dataset/seed_all/g1/csv_sonic/221125/neutral_idle_loop_002__A073
AUDIO_PATHS_CSV=${HFD_INFER_AUDIO_PATHS:-$DEFAULT_AUDIO}
ADDITIONAL_AUDIO_PATHS_CSV=${HFD_INFER_ADDITIONAL_AUDIO_PATHS:-}
INTER_TURN_BACKGROUND_STEPS=${HFD_INFER_INTER_TURN_BACKGROUND_STEPS:-0}
INIT_MOTION_DIRS_CSV=${HFD_INFER_INIT_MOTION_DIRS:-$DEFAULT_INIT_MOTION}
IFS=',' read -r -a AUDIO_PATHS <<< "$AUDIO_PATHS_CSV"
ADDITIONAL_AUDIO_PATHS=()
if [[ -n "$ADDITIONAL_AUDIO_PATHS_CSV" ]]; then
IFS=',' read -r -a ADDITIONAL_AUDIO_PATHS <<< "$ADDITIONAL_AUDIO_PATHS_CSV"
fi
IFS=',' read -r -a INIT_MOTION_DIRS <<< "$INIT_MOTION_DIRS_CSV"
if [[ ${#INIT_MOTION_DIRS[@]} -ne 1 && ${#INIT_MOTION_DIRS[@]} -ne ${#AUDIO_PATHS[@]} ]]; then
echo "[error] HFD_INFER_INIT_MOTION_DIRS must contain one path or match HFD_INFER_AUDIO_PATHS." >&2
exit 2
fi
checkpoint_ready() {
local checkpoint_dir=$1
"$PYTHON_BIN" - "$checkpoint_dir" <<'PY'
import json
import sys
from pathlib import Path
checkpoint = Path(sys.argv[1])
single = checkpoint / "model.safetensors"
manifest = checkpoint / "hfd_fused_delta_manifest.json"
boundary = checkpoint / "full_duplex_source_boundary.json"
if (
single.is_file()
and single.stat().st_size > 0
and manifest.is_file()
and manifest.stat().st_size > 0
and boundary.is_file()
and boundary.stat().st_size > 0
):
payload = json.loads(boundary.read_text(encoding="utf-8"))
if bool(payload.get("source_group_complete", False)):
raise SystemExit(0)
raise SystemExit(1)
PY
}
checkpoint_step() {
local checkpoint_name=$1
if [[ "$checkpoint_name" == "best_ckpt" ]]; then
echo best
else
echo "${checkpoint_name#ckpt_}"
fi
}
checkpoint_output_name() {
local checkpoint_dir=$1
local captured_global_step=${2:-}
local checkpoint_name
checkpoint_name=$(basename "$checkpoint_dir")
if [[ "$checkpoint_name" != "best_ckpt" ]]; then
echo "$checkpoint_name"
return
fi
if [[ -z "$captured_global_step" ]]; then
captured_global_step=$(checkpoint_global_step "$checkpoint_dir")
fi
echo "best_ckpt_step_${captured_global_step}"
}
checkpoint_global_step() {
local checkpoint_dir=$1
"$PYTHON_BIN" - "$checkpoint_dir/full_duplex_source_boundary.json" <<'PY'
import json
import sys
from pathlib import Path
marker_path = Path(sys.argv[1])
if not marker_path.is_file():
raise SystemExit(f"checkpoint has no source-boundary marker: {marker_path}")
payload = json.loads(marker_path.read_text(encoding="utf-8"))
step = int(payload["global_step"])
if step <= 0:
raise SystemExit(f"checkpoint has invalid global_step={step}")
print(step)
PY
}
safe_sample_name() {
local audio_path=$1
local index=$2
local stem
stem=$(basename "$audio_path")
stem=${stem%.*}
stem=$(echo "$stem" | tr -cs '[:alnum:]_.-' '_')
if [[ -n "$SAMPLE_TAG" ]]; then
stem="${SAMPLE_TAG}_${stem}"
fi
if [[ ${#ADDITIONAL_AUDIO_PATHS[@]} -gt 0 ]]; then
printf '%02d_%s_session%dturn' \
"$index" "$stem" "$(( ${#ADDITIONAL_AUDIO_PATHS[@]} + 1 ))"
else
printf '%02d_%s' "$index" "$stem"
fi
}
sample_output_dir() {
local checkpoint_name=$1
local sample_name=$2
if [[ -n "$SAMPLE_PARENT" ]]; then
printf '%s/%s/%s/%s\n' \
"$OUTPUT_ROOT" "$checkpoint_name" "$SAMPLE_PARENT" "$sample_name"
else
printf '%s/%s/%s\n' "$OUTPUT_ROOT" "$checkpoint_name" "$sample_name"
fi
}
render_sample() {
local sample_dir=$1
(
cd "$SONIC_RENDER_ROOT"
env -u PYTHONPATH \
PYOPENGL_PLATFORM=egl \
MUJOCO_GL=egl \
QT_X11_NO_MITSHM=1 \
SONIC_RENDER_HEADLESS_CAMERA=1 \
SONIC_RENDER_LOCAL_TMP_OUTPUT=1 \
SONIC_RENDER_TMPDIR=/tmp \
"$GMR_PY" scripts/batch_render_reference_motion_mujoco.py \
--motion-dir "$sample_dir" \
--xml-path "$GMR_XML" \
--source-fps 50 \
--target-fps "$RENDER_FPS" \
--dof-order isaaclab
)
}
process_sample() {
local checkpoint_name=$1
local step=$2
local sample_index=$3
local expected_global_step=$4
local audio_path=${AUDIO_PATHS[$sample_index]}
local init_index=$sample_index
if [[ ${#INIT_MOTION_DIRS[@]} -eq 1 ]]; then
init_index=0
fi
local init_motion_dir=${INIT_MOTION_DIRS[$init_index]}
local sample_name
sample_name=$(safe_sample_name "$audio_path" "$sample_index")
local sample_dir
sample_dir=$(sample_output_dir "$checkpoint_name" "$sample_name")
mkdir -p "$sample_dir"
if [[ ! -s "$sample_dir/infer_meta.json" || ! -s "$sample_dir/body_pos.csv" ]]; then
echo "[infer] checkpoint=$checkpoint_name sample=$sample_name audio=$audio_path"
control_args=()
additional_audio_args=()
inter_turn_args=()
for additional_audio_path in "${ADDITIONAL_AUDIO_PATHS[@]}"; do
additional_audio_args+=(--additional-audio-path "$additional_audio_path")
done
if [[ -n "$INTER_TURN_BACKGROUND_STEPS_BY_TURN" ]]; then
inter_turn_args+=(
--inter-turn-background-steps-by-turn
"$INTER_TURN_BACKGROUND_STEPS_BY_TURN"
)
fi
if [[ "$RESPECT_CONTROL" != "1" && "$RESPECT_CONTROL" != "true" ]]; then
control_args+=(--ignore-control-token)
fi
rm -f "$sample_dir/infer_failure.json"
if "$PYTHON_BIN" -u scripts/infer/full_duplex_audio_to_motion.py \
--run-dir "$RUN_DIR" \
--ckpt-step "$step" \
--output-dir "$sample_dir" \
--audio-path "$audio_path" \
"${additional_audio_args[@]}" \
--init-motion-dir "$init_motion_dir" \
--motion-frames "$MOTION_FRAMES" \
--inter-turn-background-steps "$INTER_TURN_BACKGROUND_STEPS" \
"${inter_turn_args[@]}" \
--terminal-background-steps "$TERMINAL_BACKGROUND_STEPS" \
--num-inference-steps "$INFERENCE_STEPS" \
--prompt "$PROMPT" \
--max-listen-tail-steps "$MAX_LISTEN_TAIL_STEPS" \
--initial-idle-steps "$INITIAL_IDLE_STEPS" \
--white-noise-start-index "$WHITE_NOISE_START_INDEX" \
--expected-checkpoint-global-step "$expected_global_step" \
--device "$DEVICE" \
"${control_args[@]}" 2>&1 | tee "$sample_dir/infer.log"; then
:
else
local infer_status=${PIPESTATUS[0]}
return "$infer_status"
fi
else
echo "[skip] inference exists: $sample_dir"
fi
if [[ "${HFD_INFER_SKIP_RENDER:-0}" == "1" ]]; then
touch "$sample_dir/.inferred"
echo "[inferred] queued independent rendering: $sample_dir"
return 0
fi
render_and_mux_sample "$sample_dir"
}
render_and_mux_sample() {
local sample_dir=$1
if [[ ! -f "$sample_dir/.complete" ]]; then
# The renderer validates the frame count of an existing intermediate MP4.
# A cancelled render/mux must never be accepted merely because it is nonempty.
echo "[render] sample=$sample_dir fps=$RENDER_FPS"
if ! render_sample "$sample_dir"; then
echo "[error] render failed: $sample_dir" >&2
return 1
fi
else
echo "[skip] render exists: $sample_dir/reference_motion.mp4"
fi
if [[ ! -f "$sample_dir/.complete" ]]; then
echo "[mux] use the complete background/speech/background streaming audio timeline"
if ! "$PYTHON_BIN" scripts/infer/mux_full_duplex_audio.py \
--video "$sample_dir/reference_motion.mp4" \
--audio "$sample_dir/assembled_input_audio_16k.wav" \
--output "$sample_dir/reference_motion_with_audio.mp4" \
--infer-meta "$sample_dir/infer_meta.json" \
--audio-is-timeline \
--expected-fps "$RENDER_FPS"; then
echo "[error] audio mux failed: $sample_dir" >&2
return 1
fi
else
echo "[skip] synchronized video exists: $sample_dir/reference_motion_with_audio.mp4"
fi
touch "$sample_dir/.complete"
}
if [[ -n "${HFD_RENDER_ONLY_DIR:-}" ]]; then
sample_dir=$HFD_RENDER_ONLY_DIR
if [[ "${HFD_RENDER_TASK:-}" == "swa" ]]; then
# Separate CPU-only preparation/composition; never reload a checkpoint.
"$PYTHON_BIN" scripts/infer/render_swa_result.py prepare \
--run-dir "$RUN_DIR" --output-dir "$sample_dir"
render_sample "$sample_dir/render_motion"
"$PYTHON_BIN" scripts/infer/render_swa_result.py compose \
--run-dir "$RUN_DIR" --output-dir "$sample_dir"
touch "$sample_dir/.complete"
exit 0
fi
for required_file in infer_meta.json body_pos.csv body_quat.csv joint_pos.csv assembled_input_audio_16k.wav; do
if [[ ! -s "$sample_dir/$required_file" ]]; then
echo "[error] render-only input missing: $sample_dir/$required_file" >&2
exit 1
fi
done
render_and_mux_sample "$sample_dir"
exit 0
fi
mkdir -p "$OUTPUT_ROOT"
if [[ -n "$SCENARIO_FILE" ]]; then
if [[ ! -f "$SCENARIO_FILE" ]]; then
echo "[error] HFD_INFER_SCENARIO_FILE does not exist: $SCENARIO_FILE" >&2
exit 2
fi
scenario_snapshot="$OUTPUT_ROOT/scenario.snapshot.json"
if [[ -n "$SAMPLE_PARENT" ]]; then
scenario_snapshot="$OUTPUT_ROOT/scenario.${SAMPLE_PARENT}.snapshot.json"
fi
cp -- "$SCENARIO_FILE" "$scenario_snapshot"
fi
if [[ "$MIN_CHECKPOINT_STEP" == "auto" ]]; then
MIN_CHECKPOINT_STEP=0
if [[ -z "$EXACT_CHECKPOINT_STEP" ]]; then
while IFS= read -r candidate_checkpoint; do
checkpoint_ready "$candidate_checkpoint" || continue
candidate_name=$(basename "$candidate_checkpoint")
candidate_step=${candidate_name#ckpt_}
if (( candidate_step > MIN_CHECKPOINT_STEP )); then
MIN_CHECKPOINT_STEP=$candidate_step
fi
done < <(
find "$RUN_DIR/checkpoints" -mindepth 1 -maxdepth 1 -type d \
-name 'ckpt_[0-9]*' -print 2>/dev/null
)
fi
fi
if [[ ! "$MIN_CHECKPOINT_STEP" =~ ^[0-9]+$ ]]; then
echo "[error] HFD_INFER_MIN_CHECKPOINT_STEP must be auto or a non-negative integer." >&2
exit 2
fi
echo "RUN_DIR: $RUN_DIR"
echo "OUTPUT_ROOT: $OUTPUT_ROOT"
echo "TIMING: audio 16kHz, 0.6s window, 0.4s stride; action 30/20/10 at 50fps"
echo "RENDER: ${RENDER_FPS}fps; speech follows ${INITIAL_IDLE_STEPS} idle windows; motion begins at recorded frame boundary"
echo "CONTROL: respect=$RESPECT_CONTROL initial_idle_steps=$INITIAL_IDLE_STEPS max_listen_tail_steps=$MAX_LISTEN_TAIL_STEPS white_noise_start=$WHITE_NOISE_START_INDEX"
echo "SESSION: primary_samples=${#AUDIO_PATHS[@]} additional_turns=${#ADDITIONAL_AUDIO_PATHS[@]} inter_turn_background_steps=$INTER_TURN_BACKGROUND_STEPS by_turn=${INTER_TURN_BACKGROUND_STEPS_BY_TURN:-uniform} terminal_background_steps=$TERMINAL_BACKGROUND_STEPS"
echo "CHECKPOINT POLICY: immutable ckpt_<step> only; mutable best_ckpt is never watched; min_step=$MIN_CHECKPOINT_STEP exact_step=${EXACT_CHECKPOINT_STEP:-none}"
while true; do
if [[ -n "$EXACT_CHECKPOINT_STEP" ]]; then
CHECKPOINT_DIRS=("$RUN_DIR/checkpoints/ckpt_$EXACT_CHECKPOINT_STEP")
else
mapfile -t CHECKPOINT_DIRS < <(
find "$RUN_DIR/checkpoints" -mindepth 1 -maxdepth 1 -type d \
-name 'ckpt_[0-9]*' -print 2>/dev/null | sort -V
)
fi
pending=()
for checkpoint_dir in "${CHECKPOINT_DIRS[@]}"; do
[[ -d "$checkpoint_dir" ]] || continue
checkpoint_dir_name=$(basename "$checkpoint_dir")
checkpoint_number=${checkpoint_dir_name#ckpt_}
if (( checkpoint_number < MIN_CHECKPOINT_STEP )); then
continue
fi
checkpoint_ready "$checkpoint_dir" || continue
if ! captured_global_step=$(checkpoint_global_step "$checkpoint_dir"); then
echo "[wait] checkpoint identity is not ready: $checkpoint_dir" >&2
continue
fi
checkpoint_name=$(checkpoint_output_name "$checkpoint_dir" "$captured_global_step")
settled=1
for sample_index in "${!AUDIO_PATHS[@]}"; do
sample_name=$(safe_sample_name "${AUDIO_PATHS[$sample_index]}" "$sample_index")
sample_dir=$(sample_output_dir "$checkpoint_name" "$sample_name")
if [[ ! -f "$sample_dir/.complete" && ! -f "$sample_dir/.failed" ]]; then
settled=0
break
fi
done
[[ $settled -eq 1 ]] || pending+=("$checkpoint_dir")
done
if [[ ${#pending[@]} -eq 0 ]]; then
echo "[$(date '+%F %T')] no pending checkpoint"
if [[ "$INFER_ONCE" == "1" || "$INFER_ONCE" == "true" ]]; then
break
fi
sleep "$CHECK_INTERVAL"
continue
fi
checkpoint_dir=${pending[-1]}
checkpoint_dir_name=$(basename "$checkpoint_dir")
if ! expected_global_step=$(checkpoint_global_step "$checkpoint_dir"); then
echo "[wait] selected checkpoint identity changed before inference: $checkpoint_dir" >&2
continue
fi
checkpoint_name=$(checkpoint_output_name "$checkpoint_dir" "$expected_global_step")
step=$(checkpoint_step "$checkpoint_dir_name")
echo "[$(date '+%F %T')] process newest pending checkpoint: $checkpoint_name"
for sample_index in "${!AUDIO_PATHS[@]}"; do
sample_name=$(safe_sample_name "${AUDIO_PATHS[$sample_index]}" "$sample_index")
sample_dir=$(sample_output_dir "$checkpoint_name" "$sample_name")
if [[ -f "$sample_dir/.complete" ]]; then
echo "[skip] completed sample: $sample_dir"
continue
fi
if [[ -f "$sample_dir/.failed" ]]; then
echo "[skip] failed sample already recorded: $sample_dir"
continue
fi
if process_sample \
"$checkpoint_name" \
"$step" \
"$sample_index" \
"$expected_global_step"; then
continue
else
status=$?
printf \
'checkpoint=%s\nsample=%s\nexit_status=%s\nfailed_at=%s\n' \
"$checkpoint_name" \
"$sample_name" \
"$status" \
"$(date --iso-8601=seconds)" \
> "$sample_dir/.failed"
echo "[failed] checkpoint=$checkpoint_name sample=$sample_name status=$status; details: $sample_dir/infer.log" >&2
fi
done
if [[ "$INFER_ONCE" == "1" || "$INFER_ONCE" == "true" ]]; then
break
fi
done
echo "[done] automatic full-duplex inference/render loop exited cleanly"