Download scripts/infer/run_audio_case.sh from Tinghong-Ye22/code-HFD: direct link, hf CLI and curl.
- Browser
- Download file 18.4 kB
-
https://huggingface.co/Tinghong-Ye22/code-HFD/resolve/main/scripts/infer/run_audio_case.sh
- Command line
-
hf download hf://Tinghong-Ye22/code-HFD/scripts/infer/run_audio_case.sh
-
curl -L -o run_audio_case.sh https://huggingface.co/Tinghong-Ye22/code-HFD/resolve/main/scripts/infer/run_audio_case.sh
18.4 kB
| set -Eeuo pipefail | |
| # Internal audio-case/render worker. Public entry: infer_render_auto.sh. | |
| PROJECT_ROOT=$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd) | |
| cd "$PROJECT_ROOT" | |
| export PYTHONPATH="$PROJECT_ROOT/src${PYTHONPATH:+:$PYTHONPATH}" | |
| source "$PROJECT_ROOT/scripts/activate_psi_env.sh" | |
| HFD_ENV_FILE=${HFD_ENV_FILE:-$PROJECT_ROOT/.env} | |
| if [[ -f "$HFD_ENV_FILE" ]]; then | |
| set -a | |
| source "$HFD_ENV_FILE" | |
| set +a | |
| fi | |
| PYTHON_BIN=${HFD_INFER_PYTHON:-python} | |
| RUNS_ROOT=${HFD_INFER_RUNS_ROOT:-$PROJECT_ROOT/runs} | |
| RUN_DIR=${HFD_INFER_RUN_DIR:-auto} | |
| if [[ $# -gt 0 ]]; then | |
| RUN_DIR=$1 | |
| fi | |
| if [[ "$RUN_DIR" == "auto" ]]; then | |
| RUN_DIR=$($PYTHON_BIN - "$RUNS_ROOT" <<'PY' | |
| import json | |
| import sys | |
| from pathlib import Path | |
| candidates = [] | |
| for config_path in Path(sys.argv[1]).rglob("run_config.json"): | |
| try: | |
| config = json.loads(config_path.read_text(encoding="utf-8")) | |
| except Exception: | |
| continue | |
| model = config.get("model", {}) | |
| data = config.get("data", {}) | |
| train = config.get("train", {}) | |
| if ( | |
| not model.get("streaming_qwen_enabled", False) | |
| or not data.get("online_audio", False) | |
| or train.get("full_duplex_semantics_version") not in {9, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24} | |
| ): | |
| continue | |
| candidates.append((config_path.stat().st_mtime_ns, config_path.parent)) | |
| if candidates: | |
| print(max(candidates, key=lambda item: item[0])[1]) | |
| PY | |
| ) | |
| fi | |
| if [[ -z "$RUN_DIR" || ! -f "$RUN_DIR/run_config.json" ]]; then | |
| echo "[error] Cannot find a full-duplex RUN_DIR. Pass it as the first argument." >&2 | |
| exit 2 | |
| fi | |
| RUN_DIR=$(cd "$RUN_DIR" && pwd) | |
| if [[ -n "${HFD_INFER_GPUS:-}" ]]; then | |
| export CUDA_VISIBLE_DEVICES=$HFD_INFER_GPUS | |
| elif [[ -z "${CUDA_VISIBLE_DEVICES:-}" ]]; then | |
| export CUDA_VISIBLE_DEVICES=0 | |
| fi | |
| DEVICE=${HFD_INFER_DEVICE:-cuda:0} | |
| CHECK_INTERVAL=${HFD_INFER_CHECK_INTERVAL:-30} | |
| INFER_ONCE=${HFD_INFER_ONCE:-0} | |
| MOTION_FRAMES=${HFD_INFER_MOTION_FRAMES:-450} | |
| INFERENCE_STEPS=${HFD_INFER_STEPS:-10} | |
| PROMPT=${HFD_INFER_PROMPT:-auto} | |
| RESPECT_CONTROL=${HFD_INFER_RESPECT_CONTROL:-1} | |
| MAX_LISTEN_TAIL_STEPS=${HFD_INFER_MAX_LISTEN_TAIL_STEPS:-500} | |
| INITIAL_IDLE_STEPS=${HFD_INFER_INITIAL_IDLE_STEPS:-5} | |
| WHITE_NOISE_START_INDEX=${HFD_INFER_WHITE_NOISE_START_INDEX:-3} | |
| OUTPUT_SUBDIR=${HFD_INFER_OUTPUT_SUBDIR:-full_duplex_auto_infer} | |
| OUTPUT_ROOT=${HFD_INFER_OUTPUT_ROOT:-$RUN_DIR/$OUTPUT_SUBDIR} | |
| GMR_PY=${HFD_RENDER_PYTHON:-/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/envs/gmr/bin/python} | |
| GMR_XML=${HFD_RENDER_XML:-/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/GMR/assets/unitree_g1/g1_mocap_29dof.xml} | |
| SONIC_RENDER_ROOT=${HFD_RENDER_ROOT:-/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/GR00T-WholeBodyControl/gear_sonic} | |
| RENDER_FPS=${HFD_RENDER_FPS:-50} | |
| TERMINAL_BACKGROUND_STEPS=${HFD_INFER_TERMINAL_BACKGROUND_STEPS:-0} | |
| INTER_TURN_BACKGROUND_STEPS_BY_TURN=${HFD_INFER_INTER_TURN_BACKGROUND_STEPS_BY_TURN:-} | |
| MIN_CHECKPOINT_STEP=${HFD_INFER_MIN_CHECKPOINT_STEP:-0} | |
| EXACT_CHECKPOINT_STEP=${HFD_INFER_EXACT_CHECKPOINT_STEP:-} | |
| SAMPLE_TAG=${HFD_INFER_SAMPLE_TAG:-} | |
| SAMPLE_PARENT=${HFD_INFER_SAMPLE_PARENT:-} | |
| SCENARIO_FILE=${HFD_INFER_SCENARIO_FILE:-} | |
| if [[ -n "$EXACT_CHECKPOINT_STEP" && ! "$EXACT_CHECKPOINT_STEP" =~ ^[0-9]+$ ]]; then | |
| echo "[error] HFD_INFER_EXACT_CHECKPOINT_STEP must be a non-negative integer." >&2 | |
| exit 2 | |
| fi | |
| if [[ -n "$SAMPLE_PARENT" && ! "$SAMPLE_PARENT" =~ ^[[:alnum:]_.-]+$ ]]; then | |
| echo "[error] HFD_INFER_SAMPLE_PARENT must be one safe path component." >&2 | |
| exit 2 | |
| fi | |
| # Default sample is one of the 50 fixed overfit sources (manifest row 48, | |
| # "character walks forward while grabbing something"), using the exact fixed | |
| # voice the trainer binds to that row (row_id 48 % 8 voices = female_01_en) | |
| # and a verified neutral-idle motion as the initial pose. Starting directly | |
| # from the task GT first frame is outside the training construction, which | |
| # always begins from an audited upright stationary anchor and learns a 20-frame | |
| # transition into the task. The previous default | |
| # (003771, walk back-left) is not in the overfit set: a memorizing model has | |
| # never seen a back-left walk and can only replay the forward walk it knows. | |
| DEFAULT_AUDIO=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/qwen3-tts/seed_audio/000006/female_01_en.wav | |
| DEFAULT_INIT_MOTION=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/Psi0/dataset/seed_all/g1/csv_sonic/221125/neutral_idle_loop_002__A073 | |
| AUDIO_PATHS_CSV=${HFD_INFER_AUDIO_PATHS:-$DEFAULT_AUDIO} | |
| ADDITIONAL_AUDIO_PATHS_CSV=${HFD_INFER_ADDITIONAL_AUDIO_PATHS:-} | |
| INTER_TURN_BACKGROUND_STEPS=${HFD_INFER_INTER_TURN_BACKGROUND_STEPS:-0} | |
| INIT_MOTION_DIRS_CSV=${HFD_INFER_INIT_MOTION_DIRS:-$DEFAULT_INIT_MOTION} | |
| IFS=',' read -r -a AUDIO_PATHS <<< "$AUDIO_PATHS_CSV" | |
| ADDITIONAL_AUDIO_PATHS=() | |
| if [[ -n "$ADDITIONAL_AUDIO_PATHS_CSV" ]]; then | |
| IFS=',' read -r -a ADDITIONAL_AUDIO_PATHS <<< "$ADDITIONAL_AUDIO_PATHS_CSV" | |
| fi | |
| IFS=',' read -r -a INIT_MOTION_DIRS <<< "$INIT_MOTION_DIRS_CSV" | |
| if [[ ${#INIT_MOTION_DIRS[@]} -ne 1 && ${#INIT_MOTION_DIRS[@]} -ne ${#AUDIO_PATHS[@]} ]]; then | |
| echo "[error] HFD_INFER_INIT_MOTION_DIRS must contain one path or match HFD_INFER_AUDIO_PATHS." >&2 | |
| exit 2 | |
| fi | |
| checkpoint_ready() { | |
| local checkpoint_dir=$1 | |
| "$PYTHON_BIN" - "$checkpoint_dir" <<'PY' | |
| import json | |
| import sys | |
| from pathlib import Path | |
| checkpoint = Path(sys.argv[1]) | |
| single = checkpoint / "model.safetensors" | |
| manifest = checkpoint / "hfd_fused_delta_manifest.json" | |
| boundary = checkpoint / "full_duplex_source_boundary.json" | |
| if ( | |
| single.is_file() | |
| and single.stat().st_size > 0 | |
| and manifest.is_file() | |
| and manifest.stat().st_size > 0 | |
| and boundary.is_file() | |
| and boundary.stat().st_size > 0 | |
| ): | |
| payload = json.loads(boundary.read_text(encoding="utf-8")) | |
| if bool(payload.get("source_group_complete", False)): | |
| raise SystemExit(0) | |
| raise SystemExit(1) | |
| PY | |
| } | |
| checkpoint_step() { | |
| local checkpoint_name=$1 | |
| if [[ "$checkpoint_name" == "best_ckpt" ]]; then | |
| echo best | |
| else | |
| echo "${checkpoint_name#ckpt_}" | |
| fi | |
| } | |
| checkpoint_output_name() { | |
| local checkpoint_dir=$1 | |
| local captured_global_step=${2:-} | |
| local checkpoint_name | |
| checkpoint_name=$(basename "$checkpoint_dir") | |
| if [[ "$checkpoint_name" != "best_ckpt" ]]; then | |
| echo "$checkpoint_name" | |
| return | |
| fi | |
| if [[ -z "$captured_global_step" ]]; then | |
| captured_global_step=$(checkpoint_global_step "$checkpoint_dir") | |
| fi | |
| echo "best_ckpt_step_${captured_global_step}" | |
| } | |
| checkpoint_global_step() { | |
| local checkpoint_dir=$1 | |
| "$PYTHON_BIN" - "$checkpoint_dir/full_duplex_source_boundary.json" <<'PY' | |
| import json | |
| import sys | |
| from pathlib import Path | |
| marker_path = Path(sys.argv[1]) | |
| if not marker_path.is_file(): | |
| raise SystemExit(f"checkpoint has no source-boundary marker: {marker_path}") | |
| payload = json.loads(marker_path.read_text(encoding="utf-8")) | |
| step = int(payload["global_step"]) | |
| if step <= 0: | |
| raise SystemExit(f"checkpoint has invalid global_step={step}") | |
| print(step) | |
| PY | |
| } | |
| safe_sample_name() { | |
| local audio_path=$1 | |
| local index=$2 | |
| local stem | |
| stem=$(basename "$audio_path") | |
| stem=${stem%.*} | |
| stem=$(echo "$stem" | tr -cs '[:alnum:]_.-' '_') | |
| if [[ -n "$SAMPLE_TAG" ]]; then | |
| stem="${SAMPLE_TAG}_${stem}" | |
| fi | |
| if [[ ${#ADDITIONAL_AUDIO_PATHS[@]} -gt 0 ]]; then | |
| printf '%02d_%s_session%dturn' \ | |
| "$index" "$stem" "$(( ${#ADDITIONAL_AUDIO_PATHS[@]} + 1 ))" | |
| else | |
| printf '%02d_%s' "$index" "$stem" | |
| fi | |
| } | |
| sample_output_dir() { | |
| local checkpoint_name=$1 | |
| local sample_name=$2 | |
| if [[ -n "$SAMPLE_PARENT" ]]; then | |
| printf '%s/%s/%s/%s\n' \ | |
| "$OUTPUT_ROOT" "$checkpoint_name" "$SAMPLE_PARENT" "$sample_name" | |
| else | |
| printf '%s/%s/%s\n' "$OUTPUT_ROOT" "$checkpoint_name" "$sample_name" | |
| fi | |
| } | |
| render_sample() { | |
| local sample_dir=$1 | |
| ( | |
| cd "$SONIC_RENDER_ROOT" | |
| env -u PYTHONPATH \ | |
| PYOPENGL_PLATFORM=egl \ | |
| MUJOCO_GL=egl \ | |
| QT_X11_NO_MITSHM=1 \ | |
| SONIC_RENDER_HEADLESS_CAMERA=1 \ | |
| SONIC_RENDER_LOCAL_TMP_OUTPUT=1 \ | |
| SONIC_RENDER_TMPDIR=/tmp \ | |
| "$GMR_PY" scripts/batch_render_reference_motion_mujoco.py \ | |
| --motion-dir "$sample_dir" \ | |
| --xml-path "$GMR_XML" \ | |
| --source-fps 50 \ | |
| --target-fps "$RENDER_FPS" \ | |
| --dof-order isaaclab | |
| ) | |
| } | |
| process_sample() { | |
| local checkpoint_name=$1 | |
| local step=$2 | |
| local sample_index=$3 | |
| local expected_global_step=$4 | |
| local audio_path=${AUDIO_PATHS[$sample_index]} | |
| local init_index=$sample_index | |
| if [[ ${#INIT_MOTION_DIRS[@]} -eq 1 ]]; then | |
| init_index=0 | |
| fi | |
| local init_motion_dir=${INIT_MOTION_DIRS[$init_index]} | |
| local sample_name | |
| sample_name=$(safe_sample_name "$audio_path" "$sample_index") | |
| local sample_dir | |
| sample_dir=$(sample_output_dir "$checkpoint_name" "$sample_name") | |
| mkdir -p "$sample_dir" | |
| if [[ ! -s "$sample_dir/infer_meta.json" || ! -s "$sample_dir/body_pos.csv" ]]; then | |
| echo "[infer] checkpoint=$checkpoint_name sample=$sample_name audio=$audio_path" | |
| control_args=() | |
| additional_audio_args=() | |
| inter_turn_args=() | |
| for additional_audio_path in "${ADDITIONAL_AUDIO_PATHS[@]}"; do | |
| additional_audio_args+=(--additional-audio-path "$additional_audio_path") | |
| done | |
| if [[ -n "$INTER_TURN_BACKGROUND_STEPS_BY_TURN" ]]; then | |
| inter_turn_args+=( | |
| --inter-turn-background-steps-by-turn | |
| "$INTER_TURN_BACKGROUND_STEPS_BY_TURN" | |
| ) | |
| fi | |
| if [[ "$RESPECT_CONTROL" != "1" && "$RESPECT_CONTROL" != "true" ]]; then | |
| control_args+=(--ignore-control-token) | |
| fi | |
| rm -f "$sample_dir/infer_failure.json" | |
| if "$PYTHON_BIN" -u scripts/infer/full_duplex_audio_to_motion.py \ | |
| --run-dir "$RUN_DIR" \ | |
| --ckpt-step "$step" \ | |
| --output-dir "$sample_dir" \ | |
| --audio-path "$audio_path" \ | |
| "${additional_audio_args[@]}" \ | |
| --init-motion-dir "$init_motion_dir" \ | |
| --motion-frames "$MOTION_FRAMES" \ | |
| --inter-turn-background-steps "$INTER_TURN_BACKGROUND_STEPS" \ | |
| "${inter_turn_args[@]}" \ | |
| --terminal-background-steps "$TERMINAL_BACKGROUND_STEPS" \ | |
| --num-inference-steps "$INFERENCE_STEPS" \ | |
| --prompt "$PROMPT" \ | |
| --max-listen-tail-steps "$MAX_LISTEN_TAIL_STEPS" \ | |
| --initial-idle-steps "$INITIAL_IDLE_STEPS" \ | |
| --white-noise-start-index "$WHITE_NOISE_START_INDEX" \ | |
| --expected-checkpoint-global-step "$expected_global_step" \ | |
| --device "$DEVICE" \ | |
| "${control_args[@]}" 2>&1 | tee "$sample_dir/infer.log"; then | |
| : | |
| else | |
| local infer_status=${PIPESTATUS[0]} | |
| return "$infer_status" | |
| fi | |
| else | |
| echo "[skip] inference exists: $sample_dir" | |
| fi | |
| if [[ "${HFD_INFER_SKIP_RENDER:-0}" == "1" ]]; then | |
| touch "$sample_dir/.inferred" | |
| echo "[inferred] queued independent rendering: $sample_dir" | |
| return 0 | |
| fi | |
| render_and_mux_sample "$sample_dir" | |
| } | |
| render_and_mux_sample() { | |
| local sample_dir=$1 | |
| if [[ ! -f "$sample_dir/.complete" ]]; then | |
| # The renderer validates the frame count of an existing intermediate MP4. | |
| # A cancelled render/mux must never be accepted merely because it is nonempty. | |
| echo "[render] sample=$sample_dir fps=$RENDER_FPS" | |
| if ! render_sample "$sample_dir"; then | |
| echo "[error] render failed: $sample_dir" >&2 | |
| return 1 | |
| fi | |
| else | |
| echo "[skip] render exists: $sample_dir/reference_motion.mp4" | |
| fi | |
| if [[ ! -f "$sample_dir/.complete" ]]; then | |
| echo "[mux] use the complete background/speech/background streaming audio timeline" | |
| if ! "$PYTHON_BIN" scripts/infer/mux_full_duplex_audio.py \ | |
| --video "$sample_dir/reference_motion.mp4" \ | |
| --audio "$sample_dir/assembled_input_audio_16k.wav" \ | |
| --output "$sample_dir/reference_motion_with_audio.mp4" \ | |
| --infer-meta "$sample_dir/infer_meta.json" \ | |
| --audio-is-timeline \ | |
| --expected-fps "$RENDER_FPS"; then | |
| echo "[error] audio mux failed: $sample_dir" >&2 | |
| return 1 | |
| fi | |
| else | |
| echo "[skip] synchronized video exists: $sample_dir/reference_motion_with_audio.mp4" | |
| fi | |
| touch "$sample_dir/.complete" | |
| } | |
| if [[ -n "${HFD_RENDER_ONLY_DIR:-}" ]]; then | |
| sample_dir=$HFD_RENDER_ONLY_DIR | |
| if [[ "${HFD_RENDER_TASK:-}" == "swa" ]]; then | |
| # Separate CPU-only preparation/composition; never reload a checkpoint. | |
| "$PYTHON_BIN" scripts/infer/render_swa_result.py prepare \ | |
| --run-dir "$RUN_DIR" --output-dir "$sample_dir" | |
| render_sample "$sample_dir/render_motion" | |
| "$PYTHON_BIN" scripts/infer/render_swa_result.py compose \ | |
| --run-dir "$RUN_DIR" --output-dir "$sample_dir" | |
| touch "$sample_dir/.complete" | |
| exit 0 | |
| fi | |
| for required_file in infer_meta.json body_pos.csv body_quat.csv joint_pos.csv assembled_input_audio_16k.wav; do | |
| if [[ ! -s "$sample_dir/$required_file" ]]; then | |
| echo "[error] render-only input missing: $sample_dir/$required_file" >&2 | |
| exit 1 | |
| fi | |
| done | |
| render_and_mux_sample "$sample_dir" | |
| exit 0 | |
| fi | |
| mkdir -p "$OUTPUT_ROOT" | |
| if [[ -n "$SCENARIO_FILE" ]]; then | |
| if [[ ! -f "$SCENARIO_FILE" ]]; then | |
| echo "[error] HFD_INFER_SCENARIO_FILE does not exist: $SCENARIO_FILE" >&2 | |
| exit 2 | |
| fi | |
| scenario_snapshot="$OUTPUT_ROOT/scenario.snapshot.json" | |
| if [[ -n "$SAMPLE_PARENT" ]]; then | |
| scenario_snapshot="$OUTPUT_ROOT/scenario.${SAMPLE_PARENT}.snapshot.json" | |
| fi | |
| cp -- "$SCENARIO_FILE" "$scenario_snapshot" | |
| fi | |
| if [[ "$MIN_CHECKPOINT_STEP" == "auto" ]]; then | |
| MIN_CHECKPOINT_STEP=0 | |
| if [[ -z "$EXACT_CHECKPOINT_STEP" ]]; then | |
| while IFS= read -r candidate_checkpoint; do | |
| checkpoint_ready "$candidate_checkpoint" || continue | |
| candidate_name=$(basename "$candidate_checkpoint") | |
| candidate_step=${candidate_name#ckpt_} | |
| if (( candidate_step > MIN_CHECKPOINT_STEP )); then | |
| MIN_CHECKPOINT_STEP=$candidate_step | |
| fi | |
| done < <( | |
| find "$RUN_DIR/checkpoints" -mindepth 1 -maxdepth 1 -type d \ | |
| -name 'ckpt_[0-9]*' -print 2>/dev/null | |
| ) | |
| fi | |
| fi | |
| if [[ ! "$MIN_CHECKPOINT_STEP" =~ ^[0-9]+$ ]]; then | |
| echo "[error] HFD_INFER_MIN_CHECKPOINT_STEP must be auto or a non-negative integer." >&2 | |
| exit 2 | |
| fi | |
| echo "RUN_DIR: $RUN_DIR" | |
| echo "OUTPUT_ROOT: $OUTPUT_ROOT" | |
| echo "TIMING: audio 16kHz, 0.6s window, 0.4s stride; action 30/20/10 at 50fps" | |
| echo "RENDER: ${RENDER_FPS}fps; speech follows ${INITIAL_IDLE_STEPS} idle windows; motion begins at recorded frame boundary" | |
| echo "CONTROL: respect=$RESPECT_CONTROL initial_idle_steps=$INITIAL_IDLE_STEPS max_listen_tail_steps=$MAX_LISTEN_TAIL_STEPS white_noise_start=$WHITE_NOISE_START_INDEX" | |
| echo "SESSION: primary_samples=${#AUDIO_PATHS[@]} additional_turns=${#ADDITIONAL_AUDIO_PATHS[@]} inter_turn_background_steps=$INTER_TURN_BACKGROUND_STEPS by_turn=${INTER_TURN_BACKGROUND_STEPS_BY_TURN:-uniform} terminal_background_steps=$TERMINAL_BACKGROUND_STEPS" | |
| echo "CHECKPOINT POLICY: immutable ckpt_<step> only; mutable best_ckpt is never watched; min_step=$MIN_CHECKPOINT_STEP exact_step=${EXACT_CHECKPOINT_STEP:-none}" | |
| while true; do | |
| if [[ -n "$EXACT_CHECKPOINT_STEP" ]]; then | |
| CHECKPOINT_DIRS=("$RUN_DIR/checkpoints/ckpt_$EXACT_CHECKPOINT_STEP") | |
| else | |
| mapfile -t CHECKPOINT_DIRS < <( | |
| find "$RUN_DIR/checkpoints" -mindepth 1 -maxdepth 1 -type d \ | |
| -name 'ckpt_[0-9]*' -print 2>/dev/null | sort -V | |
| ) | |
| fi | |
| pending=() | |
| for checkpoint_dir in "${CHECKPOINT_DIRS[@]}"; do | |
| [[ -d "$checkpoint_dir" ]] || continue | |
| checkpoint_dir_name=$(basename "$checkpoint_dir") | |
| checkpoint_number=${checkpoint_dir_name#ckpt_} | |
| if (( checkpoint_number < MIN_CHECKPOINT_STEP )); then | |
| continue | |
| fi | |
| checkpoint_ready "$checkpoint_dir" || continue | |
| if ! captured_global_step=$(checkpoint_global_step "$checkpoint_dir"); then | |
| echo "[wait] checkpoint identity is not ready: $checkpoint_dir" >&2 | |
| continue | |
| fi | |
| checkpoint_name=$(checkpoint_output_name "$checkpoint_dir" "$captured_global_step") | |
| settled=1 | |
| for sample_index in "${!AUDIO_PATHS[@]}"; do | |
| sample_name=$(safe_sample_name "${AUDIO_PATHS[$sample_index]}" "$sample_index") | |
| sample_dir=$(sample_output_dir "$checkpoint_name" "$sample_name") | |
| if [[ ! -f "$sample_dir/.complete" && ! -f "$sample_dir/.failed" ]]; then | |
| settled=0 | |
| break | |
| fi | |
| done | |
| [[ $settled -eq 1 ]] || pending+=("$checkpoint_dir") | |
| done | |
| if [[ ${#pending[@]} -eq 0 ]]; then | |
| echo "[$(date '+%F %T')] no pending checkpoint" | |
| if [[ "$INFER_ONCE" == "1" || "$INFER_ONCE" == "true" ]]; then | |
| break | |
| fi | |
| sleep "$CHECK_INTERVAL" | |
| continue | |
| fi | |
| checkpoint_dir=${pending[-1]} | |
| checkpoint_dir_name=$(basename "$checkpoint_dir") | |
| if ! expected_global_step=$(checkpoint_global_step "$checkpoint_dir"); then | |
| echo "[wait] selected checkpoint identity changed before inference: $checkpoint_dir" >&2 | |
| continue | |
| fi | |
| checkpoint_name=$(checkpoint_output_name "$checkpoint_dir" "$expected_global_step") | |
| step=$(checkpoint_step "$checkpoint_dir_name") | |
| echo "[$(date '+%F %T')] process newest pending checkpoint: $checkpoint_name" | |
| for sample_index in "${!AUDIO_PATHS[@]}"; do | |
| sample_name=$(safe_sample_name "${AUDIO_PATHS[$sample_index]}" "$sample_index") | |
| sample_dir=$(sample_output_dir "$checkpoint_name" "$sample_name") | |
| if [[ -f "$sample_dir/.complete" ]]; then | |
| echo "[skip] completed sample: $sample_dir" | |
| continue | |
| fi | |
| if [[ -f "$sample_dir/.failed" ]]; then | |
| echo "[skip] failed sample already recorded: $sample_dir" | |
| continue | |
| fi | |
| if process_sample \ | |
| "$checkpoint_name" \ | |
| "$step" \ | |
| "$sample_index" \ | |
| "$expected_global_step"; then | |
| continue | |
| else | |
| status=$? | |
| printf \ | |
| 'checkpoint=%s\nsample=%s\nexit_status=%s\nfailed_at=%s\n' \ | |
| "$checkpoint_name" \ | |
| "$sample_name" \ | |
| "$status" \ | |
| "$(date --iso-8601=seconds)" \ | |
| > "$sample_dir/.failed" | |
| echo "[failed] checkpoint=$checkpoint_name sample=$sample_name status=$status; details: $sample_dir/infer.log" >&2 | |
| fi | |
| done | |
| if [[ "$INFER_ONCE" == "1" || "$INFER_ONCE" == "true" ]]; then | |
| break | |
| fi | |
| done | |
| echo "[done] automatic full-duplex inference/render loop exited cleanly" | |