code-HFD / infer_render_auto.sh
Tinghong-Ye22's picture
Add files using upload-large-folder tool
aa02c28 verified
Raw History Blame Contribute Delete
5.2 kB
#!/usr/bin/env bash
set -Eeuo pipefail
PROJECT_ROOT=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
PYTHON_BIN=${HFD_INFER_PYTHON:-$PROJECT_ROOT/.venv-psi/bin/python}
# Plain bash: pin the newest compatible run once, then watch its checkpoints.
# To inspect an older run, edit RUN_DIR here or still pass it as argument 1.
RUN_DIR=${1:-${HFD_INFER_RUN_DIR:-auto}}
RUNS_ROOT=${HFD_INFER_RUNS_ROOT:-$PROJECT_ROOT/runs}
# All tasks on the first checkpoint, then every >=20k training steps. Saving
# remains every 5k. Explicit one-shot bypasses this; 0 processes every checkpoint.
export HFD_INFER_CHECKPOINT_INTERVAL=${HFD_INFER_CHECKPOINT_INTERVAL:-20000}
cd "$PROJECT_ROOT"
export PYTHONPATH="$PROJECT_ROOT/src${PYTHONPATH:+:$PYTHONPATH}"
if (( $# > 1 )); then
echo "[error] Usage: bash infer_render_auto.sh [RUN_DIR]" >&2
exit 2
fi
RUN_DIR=$("$PYTHON_BIN" "$PROJECT_ROOT/scripts/infer/run_multiturn_infer_suite.py" \
"$RUN_DIR" --runs-root "$RUNS_ROOT" --resolve-only)
echo "[HFD] pinned inference run: $RUN_DIR"
# A lightweight launch check; never allocates GPUs or scans all dataset files.
if [[ "${HFD_INFER_CONFIG_ONLY:-0}" == "1" ]]; then
echo "[HFD] inference GPUs: ${HFD_INFER_GPUS:-${CUDA_VISIBLE_DEVICES-all allocated}}"
exit 0
fi
echo "[HFD] Inference shares its visible GPUs across all tasks. Use a separate GPU allocation to avoid slowing training."
# Audio-only cases are unchanged. SWA automatically generates native answer
# speech; the legacy ``contract`` switch below only runs an extra startup check
# (it is NOT the on/off switch for SWA speech).
SPEECH_RUNTIME_MODE=${HFD_SPEECH_RUNTIME_MODE:-off}
case "$SPEECH_RUNTIME_MODE" in
off)
;;
contract)
QA_ARGS=()
if [[ -n "${HFD_QA_LORA_CHECKPOINT:-}" ]]; then
QA_ARGS+=(--qa-checkpoint "$HFD_QA_LORA_CHECKPOINT")
fi
"$PYTHON_BIN" -u "$PROJECT_ROOT/scripts/infer/validate_speech_qa_runtime.py" "${QA_ARGS[@]}"
;;
*)
echo "[error] HFD_SPEECH_RUNTIME_MODE must be off or contract (optional startup validation only)." >&2
exit 2
;;
esac
echo "[HFD] speech runtime contract=$SPEECH_RUNTIME_MODE solver=${HFD_SPEECH_SOLVER:-euler} steps=${HFD_SPEECH_SOLVER_STEPS:-6} codec_chunk=${HFD_SPEECH_CODEC_CHUNK_CODES:-36} first_chunk=${HFD_SPEECH_FIRST_CHUNK_CODES:-36} playback_buffer=${HFD_SPEECH_PLAYBACK_BUFFER_SECONDS:-0.20}s"
if [[ -z "$RUN_DIR" || ! -f "$RUN_DIR/run_config.json" ]]; then
echo "[error] A valid HFD run directory is required." >&2
exit 2
fi
# Validate the data actually saved with this run, including legacy audio-only
# runs. A newer launcher's default must not substitute a different SWA plan.
mapfile -t SAVED_SWA < <("$PYTHON_BIN" - "$RUN_DIR/run_config.json" <<'PY'
import json, sys
with open(sys.argv[1]) as file:
config = json.load(file)
print("enabled" if config.get("model", {}).get("swa_visual_qa_enabled", False) else "disabled")
print(config.get("data", {}).get("swa_source_plan", ""))
PY
)
SWA_DATA_MODE=${HFD_SWA_DATA_MODE:-${SAVED_SWA[0]}}
SWA_SOURCE_PLAN=${HFD_SWA_SOURCE_PLAN:-${SAVED_SWA[1]}}
SWA_FRONTEND_SAMPLE_COUNT=${HFD_SWA_FRONTEND_SAMPLE_COUNT:-2}
# Unified overlays must validate this run's actual manifests, not the retired
# standalone SWA source_plan/frontend cache still present for old checkpoints.
HAS_OVERLAY=$("$PYTHON_BIN" -c 'import json,sys; print(int(bool(json.load(open(sys.argv[1])).get("data",{}).get("multimodal_overlay_manifests"))))' "$RUN_DIR/run_config.json")
if [[ "$HAS_OVERLAY" == "1" ]]; then
"$PYTHON_BIN" "$PROJECT_ROOT/scripts/infer/check_run_visual_data.py" "$RUN_DIR"
else
case "$SWA_DATA_MODE" in
disabled)
echo "[HFD] SWA visual-QA inference=disabled by explicit override"
;;
contract)
if ! [[ "$SWA_FRONTEND_SAMPLE_COUNT" =~ ^[1-9][0-9]*$ ]]; then
echo "[error] HFD_SWA_FRONTEND_SAMPLE_COUNT must be a positive integer." >&2
exit 2
fi
"$PYTHON_BIN" -u "$PROJECT_ROOT/scripts/data/validate_swa_manifest.py" \
--source-plan "$SWA_SOURCE_PLAN" \
--sample-frontends "$SWA_FRONTEND_SAMPLE_COUNT" \
--require-visual \
--json
echo "[ready] SWA visual-QA contract verified"
;;
enabled)
"$PYTHON_BIN" -u "$PROJECT_ROOT/scripts/data/validate_swa_manifest.py" \
--source-plan "$SWA_SOURCE_PLAN" --sample-frontends "$SWA_FRONTEND_SAMPLE_COUNT" --require-visual --json
;;
*)
echo "[error] HFD_SWA_DATA_MODE must be disabled, contract or enabled." >&2
exit 2
;;
esac
fi
# All established interaction cases and SWA video/static train/val cases share one
# dynamic inference GPU queue (one model job per GPU). Rendering/mux use a
# separate bounded queue and never hold these model slots; default 4 workers,
# 32 outstanding videos. HFD_RENDER_WORKERS / HFD_RENDER_PENDING_LIMIT override
# these limits if needed. HFD_INCLUDE_SWA_SUITE=0 disables only SWA diagnostics.
if [[ "$SWA_DATA_MODE" == "enabled" ]]; then
export HFD_INCLUDE_SWA_SUITE=${HFD_INCLUDE_SWA_SUITE:-1}
echo "[HFD] SWA: model answer -> native Talker/Token2Wav -> independent speech worker -> video audio mix"
else
export HFD_INCLUDE_SWA_SUITE=0
fi
"$PYTHON_BIN" -u "$PROJECT_ROOT/scripts/infer/run_multiturn_infer_suite.py" "$RUN_DIR"