#!/usr/bin/env bash set -Eeuo pipefail PROJECT_ROOT=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) PYTHON_BIN=${HFD_INFER_PYTHON:-$PROJECT_ROOT/.venv-psi/bin/python} # Plain bash: pin the newest compatible run once, then watch its checkpoints. # To inspect an older run, edit RUN_DIR here or still pass it as argument 1. RUN_DIR=${1:-${HFD_INFER_RUN_DIR:-auto}} RUNS_ROOT=${HFD_INFER_RUNS_ROOT:-$PROJECT_ROOT/runs} # All tasks on the first checkpoint, then every >=20k training steps. Saving # remains every 5k. Explicit one-shot bypasses this; 0 processes every checkpoint. export HFD_INFER_CHECKPOINT_INTERVAL=${HFD_INFER_CHECKPOINT_INTERVAL:-20000} cd "$PROJECT_ROOT" export PYTHONPATH="$PROJECT_ROOT/src${PYTHONPATH:+:$PYTHONPATH}" if (( $# > 1 )); then echo "[error] Usage: bash infer_render_auto.sh [RUN_DIR]" >&2 exit 2 fi RUN_DIR=$("$PYTHON_BIN" "$PROJECT_ROOT/scripts/infer/run_multiturn_infer_suite.py" \ "$RUN_DIR" --runs-root "$RUNS_ROOT" --resolve-only) echo "[HFD] pinned inference run: $RUN_DIR" # A lightweight launch check; never allocates GPUs or scans all dataset files. if [[ "${HFD_INFER_CONFIG_ONLY:-0}" == "1" ]]; then echo "[HFD] inference GPUs: ${HFD_INFER_GPUS:-${CUDA_VISIBLE_DEVICES-all allocated}}" exit 0 fi echo "[HFD] Inference shares its visible GPUs across all tasks. Use a separate GPU allocation to avoid slowing training." # Audio-only cases are unchanged. SWA automatically generates native answer # speech; the legacy ``contract`` switch below only runs an extra startup check # (it is NOT the on/off switch for SWA speech). SPEECH_RUNTIME_MODE=${HFD_SPEECH_RUNTIME_MODE:-off} case "$SPEECH_RUNTIME_MODE" in off) ;; contract) QA_ARGS=() if [[ -n "${HFD_QA_LORA_CHECKPOINT:-}" ]]; then QA_ARGS+=(--qa-checkpoint "$HFD_QA_LORA_CHECKPOINT") fi "$PYTHON_BIN" -u "$PROJECT_ROOT/scripts/infer/validate_speech_qa_runtime.py" "${QA_ARGS[@]}" ;; *) echo "[error] HFD_SPEECH_RUNTIME_MODE must be off or contract (optional startup validation only)." >&2 exit 2 ;; esac echo "[HFD] speech runtime contract=$SPEECH_RUNTIME_MODE solver=${HFD_SPEECH_SOLVER:-euler} steps=${HFD_SPEECH_SOLVER_STEPS:-6} codec_chunk=${HFD_SPEECH_CODEC_CHUNK_CODES:-36} first_chunk=${HFD_SPEECH_FIRST_CHUNK_CODES:-36} playback_buffer=${HFD_SPEECH_PLAYBACK_BUFFER_SECONDS:-0.20}s" if [[ -z "$RUN_DIR" || ! -f "$RUN_DIR/run_config.json" ]]; then echo "[error] A valid HFD run directory is required." >&2 exit 2 fi # Validate the data actually saved with this run, including legacy audio-only # runs. A newer launcher's default must not substitute a different SWA plan. mapfile -t SAVED_SWA < <("$PYTHON_BIN" - "$RUN_DIR/run_config.json" <<'PY' import json, sys with open(sys.argv[1]) as file: config = json.load(file) print("enabled" if config.get("model", {}).get("swa_visual_qa_enabled", False) else "disabled") print(config.get("data", {}).get("swa_source_plan", "")) PY ) SWA_DATA_MODE=${HFD_SWA_DATA_MODE:-${SAVED_SWA[0]}} SWA_SOURCE_PLAN=${HFD_SWA_SOURCE_PLAN:-${SAVED_SWA[1]}} SWA_FRONTEND_SAMPLE_COUNT=${HFD_SWA_FRONTEND_SAMPLE_COUNT:-2} # Unified overlays must validate this run's actual manifests, not the retired # standalone SWA source_plan/frontend cache still present for old checkpoints. HAS_OVERLAY=$("$PYTHON_BIN" -c 'import json,sys; print(int(bool(json.load(open(sys.argv[1])).get("data",{}).get("multimodal_overlay_manifests"))))' "$RUN_DIR/run_config.json") if [[ "$HAS_OVERLAY" == "1" ]]; then "$PYTHON_BIN" "$PROJECT_ROOT/scripts/infer/check_run_visual_data.py" "$RUN_DIR" else case "$SWA_DATA_MODE" in disabled) echo "[HFD] SWA visual-QA inference=disabled by explicit override" ;; contract) if ! [[ "$SWA_FRONTEND_SAMPLE_COUNT" =~ ^[1-9][0-9]*$ ]]; then echo "[error] HFD_SWA_FRONTEND_SAMPLE_COUNT must be a positive integer." >&2 exit 2 fi "$PYTHON_BIN" -u "$PROJECT_ROOT/scripts/data/validate_swa_manifest.py" \ --source-plan "$SWA_SOURCE_PLAN" \ --sample-frontends "$SWA_FRONTEND_SAMPLE_COUNT" \ --require-visual \ --json echo "[ready] SWA visual-QA contract verified" ;; enabled) "$PYTHON_BIN" -u "$PROJECT_ROOT/scripts/data/validate_swa_manifest.py" \ --source-plan "$SWA_SOURCE_PLAN" --sample-frontends "$SWA_FRONTEND_SAMPLE_COUNT" --require-visual --json ;; *) echo "[error] HFD_SWA_DATA_MODE must be disabled, contract or enabled." >&2 exit 2 ;; esac fi # All established interaction cases and SWA video/static train/val cases share one # dynamic inference GPU queue (one model job per GPU). Rendering/mux use a # separate bounded queue and never hold these model slots; default 4 workers, # 32 outstanding videos. HFD_RENDER_WORKERS / HFD_RENDER_PENDING_LIMIT override # these limits if needed. HFD_INCLUDE_SWA_SUITE=0 disables only SWA diagnostics. if [[ "$SWA_DATA_MODE" == "enabled" ]]; then export HFD_INCLUDE_SWA_SUITE=${HFD_INCLUDE_SWA_SUITE:-1} echo "[HFD] SWA: model answer -> native Talker/Token2Wav -> independent speech worker -> video audio mix" else export HFD_INCLUDE_SWA_SUITE=0 fi "$PYTHON_BIN" -u "$PROJECT_ROOT/scripts/infer/run_multiturn_infer_suite.py" "$RUN_DIR"